Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1454 (2022-12-12 11:08:06)

Ensure that the configmap is synced before restarting processes
# Description

This ensures that the ConfigMap is synced before restarting processes (this includes the generated fdbmonitor file). The current behaviour only works because of this bug: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/611

## Type of change

*Please select one of the options below.*

- Bug fix (non-breaking change which fixes an issue)

## Discussion

-

## Testing

Unit tests and e2e tests.

## Documentation

-

## Follow-up

-



Patch:
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index 7bc83756..4524b732 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -139,6 +139,19 @@ type FoundationDBStatusProcessInfo struct {
 
 	// Roles contains a slice of all roles of the process
 	Roles []FoundationDBStatusProcessRoleInfo `json:"roles,omitempty"`
+
+	// Messages contains error messages from that fdbserver process instance
+	Messages []FoundationDBStatusProcessMessage `json:"messages,omitempty"`
+}
+
+// FoundationDBStatusProcessMessage represents an error message in the status json
+type FoundationDBStatusProcessMessage struct {
+	// Time when the error was observed
+	Time float64 `json:"time,omitempty"`
+	// The name of the error
+	Name string `json:"name,omitempty"`
+	// The type of the error
+	Type string `json:"type,omitempty"`
 }
 
 // FoundationDBStatusProcessRoleInfo contains the minimal information from the process status
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 8edf3d46..fcd6c47f 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -849,6 +849,12 @@ type FoundationDBClusterAutomationOptions struct {
 	// it will be ignored during reconciliation. This prevents that a process will block reconciliation.
 	IgnoreMissingProcessesSeconds *int `json:"ignoreMissingProcessesSeconds,omitempty"`
 
+	// FailedPodDurationSeconds defines the duration a Pod can stay in the deleted state (deletionTimestamp != 0) before
+	// it gets marked as PodFailed. This is important in cases where a fdbserver process is still reporting but the
+	// Pod resource is marked for deletion. This can happen when the kubelet or a node fails. Setting this condition
+	// will ensure that the operator is replacing affected Pods.
+	FailedPodDurationSeconds *int `json:"failedPodDurationSeconds,omitempty"`
+
 	// MaxConcurrentReplacements defines how many process groups can be concurrently
 	// replaced if they are misconfigured. If the value will be set to 0 this will block replacements
 	// and these misconfigured Pods must be replaced manually or by another process. For each reconcile
@@ -1826,13 +1832,14 @@ func (cluster *FoundationDBCluster) GetIgnoreMissingProcessesSeconds() time.Dura
 	return time.Duration(pointer.IntDeref(cluster.Spec.AutomationOptions.IgnoreMissingProcessesSeconds, 30)) * time.Second
 }
 
+// GetFailedPodDuration returns the value of FailedPodDuration or 5 minutes if unset.
+func (cluster *FoundationDBCluster) GetFailedPodDuration() time.Duration {
+	return time.Duration(pointer.IntDeref(cluster.Spec.AutomationOptions.FailedPodDurationSeconds, 300)) * time.Second
+}
+
 // GetUseNonBlockingExcludes returns the value of useNonBlockingExcludes or false if unset.
 func (cluster *FoundationDBCluster) GetUseNonBlockingExcludes() bool {
-	if cluster.Spec.AutomationOptions.UseNonBlockingExcludes == nil {
-		return false
-	}
-
-	return *cluster.Spec.AutomationOptions.UseNonBlockingExcludes
+	return pointer.BoolDeref(cluster.Spec.AutomationOptions.UseNonBlockingExcludes, false)
 }
 
 // UseLocalitiesForExclusion returns the value of UseLocalitiesForExclusion or false if unset.
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index 98e4a512..449eda63 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -495,6 +495,11 @@ func (in *FoundationDBClusterAutomationOptions) DeepCopyInto(out *FoundationDBCl
 		*out = new(int)
 		**out = **in
 	}
+	if in.FailedPodDurationSeconds != nil {
+		in, out := &in.FailedPodDurationSeconds, &out.FailedPodDurationSeconds
+		*out = new(int)
+		**out = **in
+	}
 	if in.MaxConcurrentReplacements != nil {
 		in, out := &in.MaxConcurrentReplacements, &out.MaxConcurrentReplacements
 		*out = new(int)
@@ -1132,6 +1137,11 @@ func (in *FoundationDBStatusProcessInfo) DeepCopyInto(out *FoundationDBStatusPro
 		*out = make([]FoundationDBStatusProcessRoleInfo, len(*in))
 		copy(*out, *in)
 	}
+	if in.Messages != nil {
+		in, out := &in.Messages, &out.Messages
+		*out = make([]FoundationDBStatusProcessMessage, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusProcessInfo.
@@ -1144,6 +1154,21 @@ func (in *FoundationDBStatusProcessInfo) DeepCopy() *FoundationDBStatusProcessIn
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FoundationDBStatusProcessMessage) DeepCopyInto(out *FoundationDBStatusProcessMessage) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBStatusProcessMessage.
+func (in *FoundationDBStatusProcessMessage) DeepCopy() *FoundationDBStatusProcessMessage {
+	if in == nil {
+		return nil
+	}
+	out := new(FoundationDBStatusProcessMessage)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FoundationDBStatusProcessRoleInfo) DeepCopyInto(out *FoundationDBStatusProcessRoleInfo) {
 	*out = *in
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
index 62319462..f7c4b182 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
@@ -9873,6 +9873,8 @@ spec:
                     - ProcessGroup
                     - None
                     type: string
+                  failedPodDurationSeconds:
+                    type: integer
                   ignoreMissingProcessesSeconds:
                     type: integer
                   ignorePendingPodsDuration:
diff --git a/controllers/update_status.go b/controllers/update_status.go
index 3c1d1304..a4682188 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -433,9 +433,21 @@ func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler
 		processGroup.AddAddresses(podmanager.GetPublicIPs(pod, log), processGroup.IsMarkedForRemoval() || !status.Health.Available)
 		processCount := 1
 
-		if processGroup.IsMarkedForRemoval() && pod.ObjectMeta.DeletionTimestamp != nil {
-			processGroup.UpdateCondition(fdbv1beta2.ResourcesTerminating, true, processGroups, processGroup.ProcessGroupID)
-			continue
+		// In this case the Pod has a DeletionTimestamp and should be deleted.
+		if !pod.ObjectMeta.DeletionTimestamp.IsZero() {
+			// If the ProcessGroup is marked for removal we can put the status into ResourcesTerminating
+			if processGroup.IsMarkedForRemoval() {
+				processGroup.UpdateCondition(fdbv1beta2.ResourcesTerminating, true, processGroups, processGroup.ProcessGroupID)
+				continue
+			}
+			// Otherwise we set PodFailing to ensure that the operator will trigger a replacement. This case can happen
+			// if a Pod is marked for terminating (e.g. node failure) but the process itself is still reporting to the
+			// cluster. We only set this condition if the Pod is in this state for GetFailedPodDuration(), the default
+			// here is 5 minutes.
+			if pod.ObjectMeta.DeletionTimestamp.Add(cluster.GetFailedPodDuration()).Before(time.Now()) {
+				processGroup.UpdateCondition(fdbv1beta2.PodFailing, true, processGroups, processGroup.ProcessGroupID)
+				continue
+			}
 		}
 
 		// Even the process group will be removed we need to keep the config around.
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index cf95ee49..88f3216c 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -161,6 +161,7 @@ FoundationDBClusterAutomationOptions provides flags for enabling or disabling op
 | useLocalitiesForExclusion | UseLocalitiesForExclusion defines whether the exclusions are done using localities instead of IP addresses. The default is false. | *bool | false |
 | ignoreTerminatingPodsSeconds | IgnoreTerminatingPodsSeconds defines how long a Pod has to be in the Terminating Phase before we ignore it during reconciliation. This prevents Pod that are stuck in Terminating to block further reconciliation. | *int | false |
 | ignoreMissingProcessesSeconds | IgnoreMissingProcessesSeconds defines how long a process group has to be in the MissingProcess condition until it will be ignored during reconciliation. This prevents that a process will block reconciliation. | *int | false |
+| failedPodDurationSeconds | FailedPodDurationSeconds defines the duration a Pod can stay in the deleted state (deletionTimestamp != 0) before it gets marked as PodFailed. This is important in cases where a fdbserver process is still reporting but the Pod resource is marked for deletion. This can happen when the kubelet or a node fails. Setting this condition will ensure that the operator is replacing affected Pods. | *int | false |
 | maxConcurrentReplacements | MaxConcurrentReplacements defines how many process groups can be concurrently replaced if they are misconfigured. If the value will be set to 0 this will block replacements and these misconfigured Pods must be replaced manually or by another process. For each reconcile loop the operator calculates the maximum number of possible replacements by taken this value as the upper limit and removes all ongoing replacements that have not finished. Which means if the value is set to 5 and we have 4 ongoing replacements (process groups marked with remove but not excluded) the operator is allowed to replace on further process group. | *int | false |
 | deletionMode | DeletionMode defines the deletion mode for this cluster. This can be PodUpdateModeNone, PodUpdateModeAll, PodUpdateModeZone or PodUpdateModeProcessGroup. The DeletionMode defines how Pods are deleted in order to update them or when they are removed. | [PodUpdateMode](#podupdatemode) | false |
 | removalMode | RemovalMode defines the removal mode for this cluster. This can be PodUpdateModeNone, PodUpdateModeAll, PodUpdateModeZone or PodUpdateModeProcessGroup. The RemovalMode defines how process groups are deleted in order when they are marked for removal. | [PodUpdateMode](#podupdatemode) | false |
diff --git a/go.mod b/go.mod
index c20d7bf1..0d4bcb8c 100644
--- a/go.mod
+++ b/go.mod
@@ -41,7 +41,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
+	github.com/emicklei/go-restful v2.16.0+incompatible // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
@@ -95,11 +95,11 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
 	golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/text v0.3.8 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
diff --git a/go.sum b/go.sum
index a8fb5201..e2de1c8f 100644
--- a/go.sum
+++ b/go.sum
@@ -136,8 +136,9 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epTksy9HQ1MyrbDXSdYhM=
+github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -167,7 +168,6 @@ github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSy
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -545,7 +545,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1
 github.com/vladimirvivien/gexe v0.1.1 h1:2A0SBaOSKH+cwLVdt6H+KkHZotZWRNLlWygANGw5DxE=
 github.com/vladimirvivien/gexe v0.1.1/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -595,11 +594,8 @@ go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpK
 go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
-go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI=
@@ -701,8 +697,9 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo=
+golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -794,13 +791,13 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -812,8 +809,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1060,7 +1058,6 @@ k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI
 k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU=
 k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
diff --git a/internal/restarts/restarts.go b/internal/restarts/restarts.go
index 22b7f2c2..196041de 100644
--- a/internal/restarts/restarts.go
+++ b/internal/restarts/restarts.go
@@ -31,6 +31,7 @@ func GetFilterConditions(cluster *fdbv1beta2.FoundationDBCluster) map[fdbv1beta2
 			fdbv1beta2.IncorrectCommandLine: true,
 			fdbv1beta2.IncorrectPodSpec:     false,
 			fdbv1beta2.SidecarUnreachable:   false,
+			fdbv1beta2.IncorrectConfigMap:   false,
 		}
 	}
 
@@ -40,5 +41,6 @@ func GetFilterConditions(cluster *fdbv1beta2.FoundationDBCluster) map[fdbv1beta2
 	return map[fdbv1beta2.ProcessGroupConditionType]bool{
 		fdbv1beta2.IncorrectCommandLine: true,
 		fdbv1beta2.IncorrectPodSpec:     false,
+		fdbv1beta2.IncorrectConfigMap:   false,
 	}
 }
diff --git a/kubectl-fdb/cmd/analyze.go b/kubectl-fdb/cmd/analyze.go
index 53412951..2e6bf061 100644
--- a/kubectl-fdb/cmd/analyze.go
+++ b/kubectl-fdb/cmd/analyze.go
@@ -21,7 +21,11 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"strings"
 	"time"
 
@@ -100,6 +104,17 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
+			config, err := o.configFlags.ToRESTConfig()
+			if err != nil {
+				return err
+			}
+
+			clientSet, err := kubernetes.NewForConfig(config)
+			if err != nil {
+				return err
+			}
+
+			// TODO (jscheuermann): Don't load clusters twice if we check all clusters
 			var clusters []string
 			if allClusters {
 				var clusterList fdbv1beta2.FoundationDBClusterList
@@ -116,8 +131,21 @@ func newAnalyzeCmd(streams genericclioptions.IOStreams) *cobra.Command {
 			}
 
 			var errs []error
-			for _, cluster := range clusters {
-				err := analyzeCluster(cmd, kubeClient, cluster, namespace, autoFix, wait, ignoreConditions, ignoreRemovals, sleep)
+			for _, clusterName := range clusters {
+				cluster, err := loadCluster(kubeClient, namespace, clusterName)
+				if err != nil {
+					if k8serrors.IsNotFound(err) {
+						errs = append(errs, fmt.Errorf("could not get cluster: %s/%s", namespace, clusterName))
+					}
+					errs = append(errs, err)
+				}
+
+				err = analyzeCluster(cmd, kubeClient, cluster, autoFix, wait, ignoreConditions, ignoreRemovals, sleep)
+				if err != nil {
+					errs = append(errs, err)
+				}
+
+				err = analyzeStatus(cmd, config, clientSet, kubeClient, cluster, autoFix)
 				if err != nil {
 					errs = append(errs, err)
 				}
@@ -194,12 +222,12 @@ func allConditionsValid(conditions []string) error {
 	return fmt.Errorf(errString.String())
 }
 
-type messageType string
+type messageType int
 
 const (
-	errorMessage messageType = "error"
-	warnMessage  messageType = "warn"
-	goodMessage  messageType = "good"
+	errorMessage messageType = iota
+	warnMessage
+	goodMessage
 )
 
 func printStatement(cmd *cobra.Command, line string, mesType messageType) {
@@ -222,18 +250,9 @@ func printStatement(cmd *cobra.Command, line string, mesType messageType) {
 	color.Unset()
 }
 
-func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName string, namespace string, autoFix bool, wait bool, ignoreConditions []string, ignoreRemovals bool, sleep uint16) error {
-	foundIssues := false
-	cluster, err := loadCluster(kubeClient, namespace, clusterName)
-
-	if err != nil {
-		if k8serrors.IsNotFound(err) {
-			return fmt.Errorf("could not get cluster: %s/%s", namespace, clusterName)
-		}
-		return err
-	}
-
-	cmd.Printf("Checking cluster: %s/%s\n", namespace, clusterName)
+func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, autoFix bool, wait bool, ignoreConditions []string, ignoreRemovals bool, sleep uint16) error {
+	var foundIssues bool
+	cmd.Printf("Checking cluster: %s/%s\n", cluster.Namespace, cluster.Name)
 
 	// 1. Check if the cluster is available
 	if cluster.Status.Health.Available {
@@ -325,7 +344,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 	}
 
 	// 4. Check if all Pods are up and running
-	pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+	pods, err := getPodsForCluster(kubeClient, cluster)
 	if err != nil {
 		return err
 	}
@@ -346,7 +365,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 
 		if pod.DeletionTimestamp != nil && pod.DeletionTimestamp.Add(5*time.Minute).Before(time.Now()) {
 			podIssue = true
-			statement := fmt.Sprintf("Pod %s/%s has been stuck in terminating since %s", namespace, pod.Name, pod.DeletionTimestamp)
+			statement := fmt.Sprintf("Pod %s/%s has been stuck in terminating since %s", pod.Namespace, pod.Name, pod.DeletionTimestamp)
 			printStatement(cmd, statement, errorMessage)
 
 			// The process groups that should be deleted, so we can safely replace it
@@ -359,7 +378,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 
 		if pod.Status.Phase != corev1.PodRunning {
 			podIssue = true
-			statement := fmt.Sprintf("Pod %s/%s has unexpected Phase %s with Reason: %s", namespace, pod.Name, pod.Status.Phase, pod.Status.Reason)
+			statement := fmt.Sprintf("Pod %s/%s has unexpected Phase %s with Reason: %s", pod.Namespace, pod.Name, pod.Status.Phase, pod.Status.Reason)
 			printStatement(cmd, statement, errorMessage)
 		}
 
@@ -370,7 +389,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 			}
 
 			podIssue = true
-			statement := fmt.Sprintf("Pod %s/%s has an unready container: %s", namespace, pod.Name, container.Name)
+			statement := fmt.Sprintf("Pod %s/%s has an unready container: %s", pod.Namespace, pod.Name, container.Name)
 			printStatement(cmd, statement, errorMessage)
 
 			// Replace the Pod if the container is unready for more then 30 minutes
@@ -386,7 +405,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 		id := pod.Labels[processGroupIDLabel]
 		if _, ok := processGroupMap[id]; !ok {
 			podIssue = true
-			statement := fmt.Sprintf("Pod %s/%s with the ID %s is not part of the cluster spec status", namespace, pod.Name, id)
+			statement := fmt.Sprintf("Pod %s/%s with the ID %s is not part of the cluster spec status", pod.Namespace, pod.Name, id)
 			printStatement(cmd, statement, errorMessage)
 		}
 	}
@@ -397,13 +416,12 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 		foundIssues = true
 	}
 
-	// TODO: add more checks using the FDB status directly e.g. error message or something else or overloaded or matching versions
 	// We could add more auto fixes in the future.
 	if autoFix {
 		confirmed := false
 
 		if len(failedProcessGroups) > 0 {
-			err := replaceProcessGroups(kubeClient, cluster.Name, failedProcessGroups, namespace, true, wait, false, true, sleep)
+			err := replaceProcessGroups(kubeClient, cluster.Name, failedProcessGroups, cluster.Namespace, true, wait, false, true, sleep)
 			if err != nil {
 				return err
 			}
@@ -416,7 +434,7 @@ func analyzeCluster(cmd *cobra.Command, kubeClient client.Client, clusterName st
 				podNames = append(podNames, pod.Name)
 			}
 
-			confirmed = confirmAction(fmt.Sprintf("Delete Pods %v in cluster %s/%s", strings.Join(podNames, ","), namespace, clusterName))
+			confirmed = confirmAction(fmt.Sprintf("Delete Pods %v in cluster %s/%s", strings.Join(podNames, ","), cluster.Namespace, cluster.Name))
 		}
 
 		if !wait || confirmed {
@@ -455,3 +473,101 @@ func filterDeletePods(replacements []string, killPods []corev1.Pod) []corev1.Pod
 
 	return res
 }
+
+func getStatus(restConfig *rest.Config, clientSet *kubernetes.Clientset, pod *corev1.Pod) (*fdbv1beta2.FoundationDBStatus, error) {
+	stdout, stderr, err := executeCmd(restConfig, clientSet, pod.Name, pod.Namespace, "fdbcli --exec 'status json'")
+	if err != nil {
+		return nil, fmt.Errorf("error getting status: %s, %w", stderr, err)
+	}
+
+	content, err := internal.RemoveWarningsInJSON(stdout.String())
+	if err != nil {
+		fmt.Println(stdout.String())
+		return nil, err
+	}
+
+	status := &fdbv1beta2.FoundationDBStatus{}
+	err = json.Unmarshal(content, status)
+	if err != nil {
+		return nil, err
+	}
+
+	return status, nil
+}
+
+func analyzeStatus(cmd *cobra.Command, restConfig *rest.Config, clientSet *kubernetes.Clientset, kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, autoFix bool) error {
+	cmd.Printf("Checking cluster: %s/%s with auto-fix: %t\n", cluster.Namespace, cluster.Name, autoFix)
+
+	pods, err := getPodsForCluster(kubeClient, cluster)
+	if err != nil {
+		return err
+	}
+
+	pod, err := chooseRandomPod(pods)
+	if err != nil {
+		return err
+	}
+
+	var status *fdbv1beta2.FoundationDBStatus
+	var tries int
+
+	// Try to get the status for 5 times
+	for tries < 5 {
+		status, err = getStatus(restConfig, clientSet, pod)
+		if err == nil {
+			break
+		}
+
+		tries++
+		printStatement(cmd, err.Error(), errorMessage)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return analyzeStatusInternal(cmd, restConfig, clientSet, status, pod, autoFix, cluster.Name)
+}
+
+func analyzeStatusInternal(cmd *cobra.Command, restConfig *rest.Config, clientSet *kubernetes.Clientset, status *fdbv1beta2.FoundationDBStatus, pod *corev1.Pod, autoFix bool, clusterName string) error {
+	var foundIssues bool
+
+	processesWithError := make([]string, 0)
+	for _, process := range status.Cluster.Processes {
+		if len(process.Messages) == 0 {
+			continue
+		}
+
+		foundIssues = true
+		addr := process.Address.StringWithoutFlags()
+		processesWithError = append(processesWithError, addr)
+		for _, message := range process.Messages {
+			printStatement(cmd, fmt.Sprintf("Process: %s with address: %s error: %s type: %s, time: %s",
+				process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey],
+				addr,
+				message.Name,
+				message.Type,
+				time.Unix(int64(int(message.Time)), 0).String()), errorMessage)
+		}
+	}
+
+	if len(processesWithError) > 0 && autoFix {
+		// Restart one process by another. The restart will remove the condition, assuming the condition was only
+		// intermediate.
+		for _, process := range processesWithError {
+			cmd.Println("Start killing", process)
+			killCmd := fmt.Sprintf("kill; kill %s; sleep 5; status", process)
+			_, stderr, err := executeCmd(restConfig, clientSet, pod.Name, pod.Namespace, fmt.Sprintf("fdbcli --exec '%s'", killCmd))
+			if err != nil {
+				return fmt.Errorf("error killing process %s status: %s, %w", process, stderr.String(), err)
+			}
+			time.Sleep(5 * time.Second)
+		}
+	}
+
+	if foundIssues {
+		return fmt.Errorf("found issues in status json for cluster %s. Please check them", clusterName)
+	}
+
+	return nil
+}
diff --git a/kubectl-fdb/cmd/exclusion_status.go b/kubectl-fdb/cmd/exclusion_status.go
index aeacf466..4a1f93fe 100644
--- a/kubectl-fdb/cmd/exclusion_status.go
+++ b/kubectl-fdb/cmd/exclusion_status.go
@@ -79,7 +79,7 @@ func newExclusionStatusCmd(streams genericclioptions.IOStreams) *cobra.Command {
 				return err
 			}
 
-			pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+			pods, err := getPodsForCluster(kubeClient, cluster)
 			if err != nil {
 				return err
 			}
diff --git a/kubectl-fdb/cmd/k8s_client.go b/kubectl-fdb/cmd/k8s_client.go
index c7b9c906..1d951c82 100644
--- a/kubectl-fdb/cmd/k8s_client.go
+++ b/kubectl-fdb/cmd/k8s_client.go
@@ -24,6 +24,7 @@ import (
 	"bytes"
 	ctx "context"
 	"fmt"
+	"math/rand"
 	"strings"
 
 	fdbv1beta1 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta1"
@@ -111,13 +112,13 @@ func getNodes(kubeClient client.Client, nodeSelector map[string]string) ([]strin
 	return nodes, nil
 }
 
-func getPodsForCluster(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster, namespace string) (*corev1.PodList, error) {
+func getPodsForCluster(kubeClient client.Client, cluster *fdbv1beta2.FoundationDBCluster) (*corev1.PodList, error) {
 	var podList corev1.PodList
 	err := kubeClient.List(
 		ctx.Background(),
 		&podList,
 		client.MatchingLabels(cluster.GetMatchLabels()),
-		client.InNamespace(namespace))
+		client.InNamespace(cluster.GetNamespace()))
 
 	return &podList, err
 }
@@ -178,7 +179,7 @@ func getAllPodsFromClusterWithCondition(kubeClient client.Client, clusterName st
 	}
 
 	processes := make([]string, 0, len(processesSet))
-	pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+	pods, err := getPodsForCluster(kubeClient, cluster)
 	if err != nil {
 		return processes, err
 	}
@@ -219,3 +220,20 @@ func getProcessGroupIDsFromPodName(cluster *fdbv1beta2.FoundationDBCluster, podN
 
 	return processGroupIDs, nil
 }
+
+func chooseRandomPod(pods *corev1.PodList) (*corev1.Pod, error) {
+	items := pods.Items
+	if len(items) == 0 {
+		return nil, fmt.Errorf("no pods available")
+	}
+
+	var candidate *corev1.Pod
+
+	var tries int
+	for candidate == nil || !candidate.GetDeletionTimestamp().IsZero() || tries > 10 {
+		candidate = &items[rand.Intn(len(items))]
+		tries++
+	}
+
+	return candidate, nil
+}
diff --git a/kubectl-fdb/cmd/restart.go b/kubectl-fdb/cmd/restart.go
index bac88f8d..3dd5e5f8 100644
--- a/kubectl-fdb/cmd/restart.go
+++ b/kubectl-fdb/cmd/restart.go
@@ -96,7 +96,7 @@ func newRestartCmd(streams genericclioptions.IOStreams) *cobra.Command {
 
 			var processes []string
 			if allProcesses {
-				pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+				pods, err := getPodsForCluster(kubeClient, cluster)
 				if err != nil {
 					return err
 				}
diff --git a/kubectl-fdb/cmd/root.go b/kubectl-fdb/cmd/root.go
index 49e654fd..c0701ca6 100644
--- a/kubectl-fdb/cmd/root.go
+++ b/kubectl-fdb/cmd/root.go
@@ -24,7 +24,9 @@ import (
 	"bufio"
 	"fmt"
 	"log"
+	"math/rand"
 	"os"
+	"time"
 
 	"strings"
 
@@ -51,6 +53,8 @@ func newFDBOptions(streams genericclioptions.IOStreams) *fdbBOptions {
 
 // NewRootCmd provides a cobra command wrapping FDB actions
 func NewRootCmd(streams genericclioptions.IOStreams) *cobra.Command {
+	rand.Seed(time.Now().Unix())
+
 	o := newFDBOptions(streams)
 
 	cmd := &cobra.Command{
diff --git a/mock-kubernetes-client/client/client.go b/mock-kubernetes-client/client/client.go
index aa57bcc8..f35824a8 100644
--- a/mock-kubernetes-client/client/client.go
+++ b/mock-kubernetes-client/client/client.go
@@ -48,12 +48,20 @@ type MockClient struct {
 	scheme *runtime.Scheme
 
 	// createHooks allow to inject custom logic to the creation of objects. See serviceCreateHook and podCreateHook as
-	// examples
+	// examples.
 	createHooks []func(ctx context.Context, client *MockClient, object ctrlClient.Object) error
+
+	// updateHooks allow to inject custom logic to the update of objects.
+	updateHooks []func(ctx context.Context, client *MockClient, object ctrlClient.Object) error
 }
 
 // NewMockClient creates a new MockClient.
 func NewMockClient(scheme *runtime.Scheme, hooks ...func(_ context.Context, client *MockClient, object ctrlClient.Object) error) *MockClient {
+	return NewMockClientWithHooks(scheme, hooks, nil)
+}
+
+// NewMockClientWithHooks creates a new MockClient with hooks.
+func NewMockClientWithHooks(scheme *runtime.Scheme, createHooks []func(ctx context.Context, client *MockClient, object ctrlClient.Object) error, updateHooks []func(ctx context.Context, client *MockClient, object ctrlClient.Object) error) *MockClient {
 	serviceCreateHook := func(_ context.Context, client *MockClient, object ctrlClient.Object) error {
 		svc, isSvc := object.(*corev1.Service)
 		if !isSvc {
@@ -87,7 +95,8 @@ func NewMockClient(scheme *runtime.Scheme, hooks ...func(_ context.Context, clie
 	return &MockClient{
 		fakeClient:  fake.NewClientBuilder().WithScheme(scheme).Build(),
 		scheme:      scheme,
-		createHooks: append(hooks, serviceCreateHook, podCreateHook),
+		createHooks: append(createHooks, serviceCreateHook, podCreateHook),
+		updateHooks: updateHooks,
 	}
 }
 
@@ -183,6 +192,13 @@ func (client *MockClient) Update(ctx context.Context, object ctrlClient.Object,
 		return err
 	}
 
+	for _, hook := range client.updateHooks {
+		err = hook(ctx, client, object)
+		if err != nil {
+			return err
+		}
+	}
+
 	hasSpecChanges, err := client.hasSpecChanges(existingObject, object)
 	if err != nil {
 		return err


Test patch:
diff --git a/api/v1beta2/foundationdb_status_test.go b/api/v1beta2/foundationdb_status_test.go
index 8e566319..2842f283 100644
--- a/api/v1beta2/foundationdb_status_test.go
+++ b/api/v1beta2/foundationdb_status_test.go
@@ -107,6 +107,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "log",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"c813e585043a7ab55a4905f465c4aa52": {
 							Address: ProcessAddress{
@@ -132,6 +133,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "storage",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"f9efa90fc104f4e277b140baf89aab66": {
 							Address: ProcessAddress{
@@ -157,6 +159,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "storage",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"5a633d7f4e98a6c938c84b97ec4aedbf": {
 							Address: ProcessAddress{
@@ -182,6 +185,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "log",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"5c1b68147a0ef34ce005a38245851270": {
 							Address: ProcessAddress{
@@ -207,6 +211,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "resolver",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"653defde43cf1fdef131e2fb82bd192d": {
 							Address: ProcessAddress{
@@ -241,6 +246,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "log",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 						"9c93d3b70118f16c72f7cb3f53e49f4c": {
 							Address: ProcessAddress{
@@ -269,6 +275,7 @@ var _ = Describe("FoundationDBStatus", func() {
 									Role: "storage",
 								},
 							},
+							Messages: []FoundationDBStatusProcessMessage{},
 						},
 					},
 					Data: FoundationDBStatusDataStatistics{
@@ -497,6 +504,7 @@ var _ = Describe("FoundationDBStatus", func() {
 						{Role: "grv_proxy"},
 						{Role: "storage"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"eab0db1aa7aae81a50ca97e9814a1b7d": {
 					Address: ProcessAddress{
@@ -519,6 +527,7 @@ var _ = Describe("FoundationDBStatus", func() {
 						{Role: string(ProcessClassStorage)},
 						{Role: "resolver"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"f483247d4d5f279ef02c549680cbde64": {
 					Address: ProcessAddress{
@@ -541,6 +550,7 @@ var _ = Describe("FoundationDBStatus", func() {
 						{Role: "commit_proxy"},
 						{Role: "storage"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"f6e0f7fd80da429d20329ad95d793ca3": {
 					Address: ProcessAddress{
@@ -563,6 +573,7 @@ var _ = Describe("FoundationDBStatus", func() {
 						{Role: "data_distributor"},
 						{Role: "ratekeeper"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"f75644abdf1b06c803b5c3c124fdd0a0": {
 					Address: ProcessAddress{
@@ -583,6 +594,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{Role: string(ProcessClassClusterController)},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"105bf6c041f8ec315d03e889c2746ecf": {
 					Address: ProcessAddress{
@@ -603,6 +615,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{Role: "log"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"78c1c84af4481f0df628d40358f0930a": {
 					Address: ProcessAddress{
@@ -623,6 +636,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{Role: "log"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 				"83084479b50c9c3a09b0286297be3796": {
 					Address: ProcessAddress{
@@ -643,6 +657,7 @@ var _ = Describe("FoundationDBStatus", func() {
 					Roles: []FoundationDBStatusProcessRoleInfo{
 						{Role: "log"},
 					},
+					Messages: []FoundationDBStatusProcessMessage{},
 				},
 			},
 			Data: FoundationDBStatusDataStatistics{
diff --git a/controllers/update_status_test.go b/controllers/update_status_test.go
index 2a9c0b38..efbc7c36 100644
--- a/controllers/update_status_test.go
+++ b/controllers/update_status_test.go
@@ -22,6 +22,8 @@ package controllers
 
 import (
 	"context"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"time"
 
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/fdbadminclient/mock"
 
@@ -191,6 +193,32 @@ var _ = Describe("update_status", func() {
 			})
 		})
 
+		When("the Pod is marked for deletion but still reporting to the cluster", func() {
+			BeforeEach(func() {
+				// We cannot use the k8sClient.MockStuckTermination() method because the deletionTimestamp must be
+				// older than 5 minutes to detect the Pod as PodFailed.
+				stuckPod := pods[0]
+				stuckPod.SetDeletionTimestamp(&metav1.Time{Time: time.Now().Add(-10 * time.Minute)})
+				stuckPod.SetFinalizers(append(stuckPod.GetFinalizers(), "foundationdb.org/testing"))
+				Expect(k8sClient.Update(context.Background(), stuckPod)).NotTo(HaveOccurred())
+			})
+
+			It("should get a condition assigned", func() {
+				// Ensure that we have the same number of Pods and processes
+				Expect(allPods).To(HaveLen(len(processMap)))
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs)
+				Expect(err).NotTo(HaveOccurred())
+
+				missingProcesses := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.PodFailing, false)
+				Expect(missingProcesses).To(Equal([]string{"storage-1"}))
+
+				Expect(len(processGroupStatus)).To(BeNumerically(">", 4))
+				processGroup := processGroupStatus[len(processGroupStatus)-4]
+				Expect(processGroup.ProcessGroupID).To(Equal("storage-1"))
+				Expect(len(processGroup.ProcessGroupConditions)).To(Equal(1))
+			})
+		})
+
 		When("the pod is failing to launch", func() {
 			BeforeEach(func() {
 				pods[0].Status.ContainerStatuses[0].Ready = false
diff --git a/internal/restarts/restarts_test.go b/internal/restarts/restarts_test.go
index 90f4eafa..6a6cbef9 100644
--- a/internal/restarts/restarts_test.go
+++ b/internal/restarts/restarts_test.go
@@ -43,6 +43,7 @@ var _ = Describe("restarts", func() {
 				fdbv1beta2.IncorrectCommandLine: true,
 				fdbv1beta2.IncorrectPodSpec:     false,
 				fdbv1beta2.SidecarUnreachable:   false,
+				fdbv1beta2.IncorrectConfigMap:   false,
 			}),
 		Entry("when the running version is missing",
 			&fdbv1beta2.FoundationDBCluster{
@@ -54,6 +55,7 @@ var _ = Describe("restarts", func() {
 				fdbv1beta2.IncorrectCommandLine: true,
 				fdbv1beta2.IncorrectPodSpec:     false,
 				fdbv1beta2.SidecarUnreachable:   false,
+				fdbv1beta2.IncorrectConfigMap:   false,
 			}),
 		Entry("when an upgrade is performed",
 			&fdbv1beta2.FoundationDBCluster{
@@ -67,6 +69,7 @@ var _ = Describe("restarts", func() {
 			map[fdbv1beta2.ProcessGroupConditionType]bool{
 				fdbv1beta2.IncorrectCommandLine: true,
 				fdbv1beta2.IncorrectPodSpec:     false,
+				fdbv1beta2.IncorrectConfigMap:   false,
 			}),
 	)
 })
diff --git a/kubectl-fdb/cmd/analyze_test.go b/kubectl-fdb/cmd/analyze_test.go
index 6883b7b8..b4522449 100644
--- a/kubectl-fdb/cmd/analyze_test.go
+++ b/kubectl-fdb/cmd/analyze_test.go
@@ -40,7 +40,7 @@ func getCluster(clusterName string, namespace string, available bool, healthy bo
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       clusterName,
 			Namespace:  namespace,
-			Generation: 0,
+			Generation: 1,
 		},
 		Spec: fdbv1beta2.FoundationDBClusterSpec{
 			ProcessCounts: fdbv1beta2.ProcessCounts{
@@ -97,8 +97,6 @@ var _ = Describe("[plugin] analyze cluster", func() {
 
 		DescribeTable("return all successful and failed checks",
 			func(tc testCase) {
-				Expect(k8sClient.Delete(context.TODO(), tc.cluster)).NotTo(HaveOccurred())
-				Expect(k8sClient.Create(context.TODO(), tc.cluster)).NotTo(HaveOccurred())
 				for _, pod := range tc.podList.Items {
 					Expect(k8sClient.Create(context.TODO(), &pod)).NotTo(HaveOccurred())
 				}
@@ -109,7 +107,7 @@ var _ = Describe("[plugin] analyze cluster", func() {
 				inBuffer := bytes.Buffer{}
 
 				cmd := newAnalyzeCmd(genericclioptions.IOStreams{In: &inBuffer, Out: &outBuffer, ErrOut: &errBuffer})
-				err := analyzeCluster(cmd, k8sClient, clusterName, namespace, tc.AutoFix, tc.NoWait, tc.IgnoredConditions, tc.IgnoreRemovals, 0)
+				err := analyzeCluster(cmd, k8sClient, tc.cluster, tc.AutoFix, tc.NoWait, tc.IgnoredConditions, tc.IgnoreRemovals, 0)
 
 				if err != nil && !tc.HasErrors {
 					Expect(err).To(HaveOccurred())
diff --git a/mock-kubernetes-client/client/client_test.go b/mock-kubernetes-client/client/client_test.go
index 4657c9da..8523f1b3 100644
--- a/mock-kubernetes-client/client/client_test.go
+++ b/mock-kubernetes-client/client/client_test.go
@@ -62,7 +62,7 @@ func createDummyPod() *corev1.Pod {
 }
 
 var _ = Describe("[mock client]", func() {
-	var client *MockClient
+	var mockClient *MockClient
 
 	BeforeEach(func() {
 		err := scheme.AddToScheme(scheme.Scheme)
@@ -70,18 +70,18 @@ var _ = Describe("[mock client]", func() {
 		err = fdbv1beta2.AddToScheme(scheme.Scheme)
 		Expect(err).NotTo(HaveOccurred())
 
-		client = NewMockClient(scheme.Scheme)
+		mockClient = NewMockClient(scheme.Scheme)
 	})
 
 	When("creating and getting an object", func() {
 		It("should create and get the object", func() {
 			pod := createDummyPod()
-			err := client.Create(context.TODO(), pod)
+			err := mockClient.Create(context.TODO(), pod)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(pod.ObjectMeta.Generation).To(Equal(int64(1)))
 
 			podCopy := &corev1.Pod{}
-			err = client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, podCopy)
+			err = mockClient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, podCopy)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(podCopy.Name).To(Equal("pod1"))
 			Expect(len(podCopy.Spec.Containers)).To(Equal(1))
@@ -95,13 +95,13 @@ var _ = Describe("[mock client]", func() {
 
 		BeforeEach(func() {
 			initialPod = createDummyPod()
-			err := client.Create(context.TODO(), initialPod)
+			err := mockClient.Create(context.TODO(), initialPod)
 			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("should create a mock IP", func() {
 			pod := &corev1.Pod{}
-			err := client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, pod)
+			err := mockClient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, pod)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(pod.Name).To(Equal("pod1"))
 			Expect(len(pod.Spec.Containers)).To(Equal(1))
@@ -112,13 +112,13 @@ var _ = Describe("[mock client]", func() {
 
 		When("Removing the Pod IP", func() {
 			BeforeEach(func() {
-				err := client.RemovePodIP(initialPod)
+				err := mockClient.RemovePodIP(initialPod)
 				Expect(err).NotTo(HaveOccurred())
 			})
 
 			It("should return an empty Pod IP", func() {
 				pod := &corev1.Pod{}
-				err := client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, pod)
+				err := mockClient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, pod)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(pod.Name).To(Equal("pod1"))
 				Expect(len(pod.Spec.Containers)).To(Equal(1))
@@ -131,10 +131,10 @@ var _ = Describe("[mock client]", func() {
 
 	When("creating and getting twice object", func() {
 		It("should return an error", func() {
-			err := client.Create(context.TODO(), createDummyPod())
+			err := mockClient.Create(context.TODO(), createDummyPod())
 			Expect(err).NotTo(HaveOccurred())
 
-			err = client.Create(context.TODO(), createDummyPod())
+			err = mockClient.Create(context.TODO(), createDummyPod())
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(Equal("pods \"pod1\" already exists"))
 		})
@@ -148,7 +148,7 @@ var _ = Describe("[mock client]", func() {
 					Name:      "service1",
 				},
 			}
-			err := client.Create(context.TODO(), service)
+			err := mockClient.Create(context.TODO(), service)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(service.ObjectMeta.Generation).To(Equal(int64(1)))
 			Expect(service.Spec.ClusterIP).To(Equal("192.168.0.1"))
@@ -162,7 +162,7 @@ var _ = Describe("[mock client]", func() {
 					ClusterIP: "None",
 				},
 			}
-			err = client.Create(context.TODO(), service)
+			err = mockClient.Create(context.TODO(), service)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(service.ObjectMeta.Generation).To(Equal(int64(1)))
 			Expect(service.Spec.ClusterIP).To(Equal("None"))
@@ -171,16 +171,16 @@ var _ = Describe("[mock client]", func() {
 
 	When("getting a missing object", func() {
 		It("return an is not found error", func() {
-			err := client.Create(context.TODO(), createDummyPod())
+			err := mockClient.Create(context.TODO(), createDummyPod())
 			Expect(err).NotTo(HaveOccurred())
 
 			pod := &corev1.Pod{}
-			err = client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod2"}, pod)
+			err = mockClient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod2"}, pod)
 			Expect(err).To(HaveOccurred())
 			Expect(k8serrors.IsNotFound(err)).To(BeTrue())
 
 			deployment := &appsv1.Deployment{}
-			err = client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, deployment)
+			err = mockClient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, deployment)
 			Expect(err).To(HaveOccurred())
 			Expect(k8serrors.IsNotFound(err)).To(BeTrue())
 		})
@@ -189,19 +189,19 @@ var _ = Describe("[mock client]", func() {
 	When("deleting an object", func() {
 		It("it should be able to delete the object or return an error", func() {
 			pod := createDummyPod()
-			err := client.Create(context.TODO(), pod)
+			err := mockClient.Create(context.TODO(), pod)
 			Expect(err).NotTo(HaveOccurred())
 
 			objectKey := types.NamespacedName{Namespace: "default", Name: "pod1"}
-			err = client.Delete(context.TODO(), pod)
+			err = mockClient.Delete(context.TODO(), pod)
 			Expect(err).NotTo(HaveOccurred())
 
 			podCopy := &corev1.Pod{}
-			err = client.Get(context.TODO(), objectKey, podCopy)
+			err = mockClient.Get(context.TODO(), objectKey, podCopy)
 			Expect(err).To(HaveOccurred())
 			Expect(k8serrors.IsNotFound(err)).To(BeTrue())
 
-			err = client.Delete(context.TODO(), pod)
+			err = mockClient.Delete(context.TODO(), pod)
 			Expect(err).To(HaveOccurred())
 			Expect(k8serrors.IsNotFound(err)).To(BeTrue())
 		})
@@ -213,25 +213,25 @@ var _ = Describe("[mock client]", func() {
 			pod2 := createDummyPod()
 			pod2.Name = "pod2"
 
-			err := client.Create(context.TODO(), pod1)
+			err := mockClient.Create(context.TODO(), pod1)
 			Expect(err).NotTo(HaveOccurred())
-			err = client.Create(context.TODO(), pod2)
+			err = mockClient.Create(context.TODO(), pod2)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = client.MockStuckTermination(pod1, true)
+			err = mockClient.MockStuckTermination(pod1, true)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = client.Delete(context.TODO(), pod1)
+			err = mockClient.Delete(context.TODO(), pod1)
 			Expect(err).NotTo(HaveOccurred())
 
 			podCopy := &corev1.Pod{}
-			err = client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, podCopy)
+			err = mockClient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, podCopy)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = client.Delete(context.TODO(), pod2)
+			err = mockClient.Delete(context.TODO(), pod2)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod2"}, podCopy)
+			err = mockClient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod2"}, podCopy)
 			Expect(err).To(HaveOccurred())
 			Expect(k8serrors.IsNotFound(err)).To(BeTrue())
 		})
@@ -240,25 +240,25 @@ var _ = Describe("[mock client]", func() {
 	When("updating an object", func() {
 		It("the object should be updated", func() {
 			pod := createDummyPod()
-			err := client.Create(context.TODO(), pod)
+			err := mockClient.Create(context.TODO(), pod)
 			Expect(err).NotTo(HaveOccurred())
 
 			container := &pod.Spec.Containers[0]
 			container.Env = append(container.Env, corev1.EnvVar{Name: "test-env"})
 
-			err = client.Update(context.TODO(), pod)
+			err = mockClient.Update(context.TODO(), pod)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(pod.ObjectMeta.Generation).To(Equal(int64(2)))
 
 			podCopy := &corev1.Pod{}
-			err = client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, podCopy)
+			err = mockClient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, podCopy)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(podCopy.Spec.Containers)).To(Equal(1))
 			Expect(len(podCopy.Spec.Containers[0].Env)).To(Equal(1))
 			Expect(podCopy.Spec.Containers[0].Env[0].Name).To(Equal("test-env"))
 			Expect(podCopy.ObjectMeta.Generation).To(Equal(int64(2)))
 
-			err = client.Update(context.TODO(), pod)
+			err = mockClient.Update(context.TODO(), pod)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(pod.ObjectMeta.Generation).To(Equal(int64(2)))
 		})
@@ -267,17 +267,17 @@ var _ = Describe("[mock client]", func() {
 	When("updating the object status", func() {
 		It("should update the object status", func() {
 			pod := createDummyPod()
-			err := client.Create(context.TODO(), pod)
+			err := mockClient.Create(context.TODO(), pod)
 			Expect(err).NotTo(HaveOccurred())
 
 			pod.Status.HostIP = "foo"
 
-			err = client.Status().Update(context.TODO(), pod)
+			err = mockClient.Status().Update(context.TODO(), pod)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(pod.ObjectMeta.Generation).To(Equal(int64(1)))
 
 			podCopy := &corev1.Pod{}
-			err = client.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, podCopy)
+			err = mockClient.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: "pod1"}, podCopy)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(podCopy.Status.HostIP).To(Equal("foo"))
 			Expect(podCopy.ObjectMeta.Generation).To(Equal(int64(1)))
@@ -290,13 +290,13 @@ var _ = Describe("[mock client]", func() {
 			pod2 := createDummyPod()
 			pod2.Name = "pod2"
 
-			err := client.Create(context.TODO(), pod1)
+			err := mockClient.Create(context.TODO(), pod1)
 			Expect(err).NotTo(HaveOccurred())
-			err = client.Create(context.TODO(), pod2)
+			err = mockClient.Create(context.TODO(), pod2)
 			Expect(err).NotTo(HaveOccurred())
 
 			pods := &corev1.PodList{}
-			err = client.List(context.TODO(), pods)
+			err = mockClient.List(context.TODO(), pods)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(pods.Items)).To(Equal(2))
 
@@ -312,17 +312,17 @@ var _ = Describe("[mock client]", func() {
 			pod2 := createDummyPod()
 			pod2.Name = "pod2"
 
-			err := client.Create(context.TODO(), pod1)
+			err := mockClient.Create(context.TODO(), pod1)
 			Expect(err).NotTo(HaveOccurred())
-			err = client.Create(context.TODO(), pod2)
+			err = mockClient.Create(context.TODO(), pod2)
 			Expect(err).NotTo(HaveOccurred())
 
 			pods := &corev1.PodList{}
-			err = client.List(context.TODO(), pods, ctrlClient.InNamespace("default"))
+			err = mockClient.List(context.TODO(), pods, ctrlClient.InNamespace("default"))
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(pods.Items)).To(Equal(2))
 
-			err = client.List(context.TODO(), pods, ctrlClient.InNamespace("not-default"))
+			err = mockClient.List(context.TODO(), pods, ctrlClient.InNamespace("not-default"))
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(pods.Items)).To(Equal(0))
 		})
@@ -338,15 +338,15 @@ var _ = Describe("[mock client]", func() {
 			pod3.Name = "pod3"
 			pod3.Labels = nil
 
-			err := client.Create(context.TODO(), pod1)
+			err := mockClient.Create(context.TODO(), pod1)
 			Expect(err).NotTo(HaveOccurred())
-			err = client.Create(context.TODO(), pod2)
+			err = mockClient.Create(context.TODO(), pod2)
 			Expect(err).NotTo(HaveOccurred())
-			err = client.Create(context.TODO(), pod3)
+			err = mockClient.Create(context.TODO(), pod3)
 			Expect(err).NotTo(HaveOccurred())
 
 			pods := &corev1.PodList{}
-			err = client.List(context.TODO(), pods, ctrlClient.MatchingLabels(map[string]string{"app": "app2"}))
+			err = mockClient.List(context.TODO(), pods, ctrlClient.MatchingLabels(map[string]string{"app": "app2"}))
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(pods.Items)).To(Equal(1))
 			Expect(pods.Items[0].Name).To(Equal("pod2"))
@@ -354,7 +354,7 @@ var _ = Describe("[mock client]", func() {
 			appRequirement, err := labels.NewRequirement("app", selection.Exists, nil)
 			Expect(err).NotTo(HaveOccurred())
 			pods = &corev1.PodList{}
-			err = client.List(context.TODO(), pods, ctrlClient.MatchingLabelsSelector{Selector: labels.NewSelector().Add(*appRequirement)})
+			err = mockClient.List(context.TODO(), pods, ctrlClient.MatchingLabelsSelector{Selector: labels.NewSelector().Add(*appRequirement)})
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(pods.Items)).To(Equal(2))
 			sortPodsByName(pods)
@@ -372,13 +372,13 @@ var _ = Describe("[mock client]", func() {
 	//		pod2 := createDummyPod()
 	//		pod2.Name = "pod2"
 	//
-	//		err := client.Create(context.TODO(), pod1)
+	//		err := mockClient.Create(context.TODO(), pod1)
 	//		Expect(err).NotTo(HaveOccurred())
-	//		err = client.Create(context.TODO(), pod2)
+	//		err = mockClient.Create(context.TODO(), pod2)
 	//		Expect(err).NotTo(HaveOccurred())
 	//
 	//		pods := &corev1.PodList{}
-	//		err = client.List(context.TODO(), pods, ctrlClient.MatchingFields{"metadata.name": "pod1"})
+	//		err = mockClient.List(context.TODO(), pods, ctrlClient.MatchingFields{"metadata.name": "pod1"})
 	//		Expect(err).NotTo(HaveOccurred())
 	//		Expect(len(pods.Items)).To(Equal(1))
 	//		Expect(pods.Items[0].Name).To(Equal("pod1"))
@@ -389,9 +389,9 @@ var _ = Describe("[mock client]", func() {
 		It("should create the event", func() {
 			pod := createDummyPod()
 			pod.ObjectMeta.UID = uuid.NewUUID()
-			client.Event(pod, "Testing", "This is a test", "Test message")
+			mockClient.Event(pod, "Testing", "This is a test", "Test message")
 			events := &corev1.EventList{}
-			err := client.List(context.TODO(), events)
+			err := mockClient.List(context.TODO(), events)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(events.Items)).To(Equal(1))
 			Expect(events.Items[0].Namespace).To(Equal("default"))
@@ -408,9 +408,9 @@ var _ = Describe("[mock client]", func() {
 
 	When("creating an event with a message format", func() {
 		It("should create the event", func() {
-			client.Eventf(createDummyPod(), "Testing", "This is a test", "Test message: %d", 5)
+			mockClient.Eventf(createDummyPod(), "Testing", "This is a test", "Test message: %d", 5)
 			events := &corev1.EventList{}
-			err := client.List(context.TODO(), events)
+			err := mockClient.List(context.TODO(), events)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(events.Items)).To(Equal(1))
 			Expect(events.Items[0].Namespace).To(Equal("default"))
@@ -426,9 +426,9 @@ var _ = Describe("[mock client]", func() {
 	When("creating an event with in the past", func() {
 		It("should create the event", func() {
 			timestamp := metav1.Time{Time: time.Now().Add(-1 * time.Minute)}
-			client.PastEventf(createDummyPod(), timestamp, "Testing", "This is a test", "Test message: %d", 5)
+			mockClient.PastEventf(createDummyPod(), timestamp, "Testing", "This is a test", "Test message: %d", 5)
 			events := &corev1.EventList{}
-			err := client.List(context.TODO(), events)
+			err := mockClient.List(context.TODO(), events)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(events.Items)).To(Equal(1))
 			Expect(events.Items[0].Namespace).To(Equal("default"))
@@ -444,9 +444,9 @@ var _ = Describe("[mock client]", func() {
 
 	When("creating an event with annotations", func() {
 		It("should create the event", func() {
-			client.AnnotatedEventf(createDummyPod(), map[string]string{"anno": "value"}, "Testing", "This is a test", "Test message: %d", 5)
+			mockClient.AnnotatedEventf(createDummyPod(), map[string]string{"anno": "value"}, "Testing", "This is a test", "Test message: %d", 5)
 			events := &corev1.EventList{}
-			err := client.List(context.TODO(), events)
+			err := mockClient.List(context.TODO(), events)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(events.Items)).To(Equal(1))
 			Expect(events.Items[0].Namespace).To(Equal("default"))
@@ -464,7 +464,7 @@ var _ = Describe("[mock client]", func() {
 		var expectedPods = int32(4)
 
 		BeforeEach(func() {
-			client = NewMockClient(scheme.Scheme, func(ctx context.Context, client *MockClient, object ctrlClient.Object) error {
+			mockClient = NewMockClient(scheme.Scheme, func(ctx context.Context, client *MockClient, object ctrlClient.Object) error {
 				replicaSet, isReplicaSet := object.(*appsv1.ReplicaSet)
 				if !isReplicaSet {
 					return nil
@@ -489,7 +489,7 @@ var _ = Describe("[mock client]", func() {
 				return nil
 			})
 
-			Expect(client.Create(context.TODO(), &appsv1.ReplicaSet{
+			Expect(mockClient.Create(context.TODO(), &appsv1.ReplicaSet{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "unicorn",
 				},
@@ -501,16 +501,106 @@ var _ = Describe("[mock client]", func() {
 
 		It("should run the webhook and create 4 Pods", func() {
 			podList := &corev1.PodList{}
-			Expect(client.List(context.TODO(), podList)).NotTo(HaveOccurred())
+			Expect(mockClient.List(context.TODO(), podList)).NotTo(HaveOccurred())
 			Expect(podList.Items).To(HaveLen(int(expectedPods)))
 
 			// Ensure the label is update
 			replicaSet := &appsv1.ReplicaSet{}
-			Expect(client.Get(context.TODO(), ctrlClient.ObjectKey{Name: "unicorn"}, replicaSet)).NotTo(HaveOccurred())
+			Expect(mockClient.Get(context.TODO(), ctrlClient.ObjectKey{Name: "unicorn"}, replicaSet)).NotTo(HaveOccurred())
 			Expect(pointer.Int32Deref(replicaSet.Spec.Replicas, -1)).To(BeNumerically("==", expectedPods))
 			Expect(replicaSet.Labels).To(HaveKeyWithValue("test", "success"))
 		})
 	})
+
+	When("adding a custom create and update hook", func() {
+		var expectedPods = int32(4)
+		var additionalPods = int32(12)
+
+		BeforeEach(func() {
+			createHook := func(ctx context.Context, client *MockClient, object ctrlClient.Object) error {
+				replicaSet, isReplicaSet := object.(*appsv1.ReplicaSet)
+				if !isReplicaSet {
+					return nil
+				}
+
+				expectedReplicas := pointer.Int32Deref(replicaSet.Spec.Replicas, 1)
+
+				for i := int32(0); i < expectedReplicas; i++ {
+					err := client.Create(ctx, &corev1.Pod{
+						ObjectMeta: metav1.ObjectMeta{
+							Namespace: replicaSet.Namespace,
+							Name:      fmt.Sprintf("%s-%d", replicaSet.Name, i),
+						},
+					})
+					_ = err
+				}
+
+				replicaSet.Labels = map[string]string{
+					"test": "success",
+				}
+
+				return nil
+			}
+
+			updateHook := func(ctx context.Context, client *MockClient, object ctrlClient.Object) error {
+				replicaSet, isReplicaSet := object.(*appsv1.ReplicaSet)
+				if !isReplicaSet {
+					return nil
+				}
+
+				podList := &corev1.PodList{}
+				Expect(client.List(context.TODO(), podList)).NotTo(HaveOccurred())
+
+				expectedReplicas := pointer.Int32Deref(replicaSet.Spec.Replicas, 1)
+				for i := int32(len(podList.Items)); i < expectedReplicas; i++ {
+					err := client.Create(ctx, &corev1.Pod{
+						ObjectMeta: metav1.ObjectMeta{
+							Namespace: replicaSet.Namespace,
+							Name:      fmt.Sprintf("%s-%d", replicaSet.Name, i),
+						},
+					})
+					_ = err
+				}
+
+				replicaSet.Labels = map[string]string{
+					"test": "success",
+				}
+
+				return nil
+			}
+
+			mockClient = NewMockClientWithHooks(scheme.Scheme, []func(ctx context.Context, client *MockClient, object ctrlClient.Object) error{
+				createHook,
+			}, []func(ctx context.Context, client *MockClient, object ctrlClient.Object) error{
+				updateHook,
+			})
+
+			replicaSet := &appsv1.ReplicaSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "unicorn",
+				},
+				Spec: appsv1.ReplicaSetSpec{
+					Replicas: pointer.Int32(expectedPods),
+				},
+			}
+			Expect(mockClient.Create(context.TODO(), replicaSet)).NotTo(HaveOccurred())
+
+			replicaSet.Spec.Replicas = pointer.Int32(expectedPods + additionalPods)
+			Expect(mockClient.Update(context.TODO(), replicaSet)).NotTo(HaveOccurred())
+		})
+
+		It("should run the webhook and create 16 Pods", func() {
+			podList := &corev1.PodList{}
+			Expect(mockClient.List(context.TODO(), podList)).NotTo(HaveOccurred())
+			Expect(podList.Items).To(HaveLen(int(expectedPods + additionalPods)))
+
+			// Ensure the label is update
+			replicaSet := &appsv1.ReplicaSet{}
+			Expect(mockClient.Get(context.TODO(), ctrlClient.ObjectKey{Name: "unicorn"}, replicaSet)).NotTo(HaveOccurred())
+			Expect(pointer.Int32Deref(replicaSet.Spec.Replicas, -1)).To(BeNumerically("==", expectedPods+additionalPods))
+			Expect(replicaSet.Labels).To(HaveKeyWithValue("test", "success"))
+		})
+	})
 })
 
 func sortPodsByName(pods *corev1.PodList) {