Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1637 (2023-05-17 11:27:30)

Remove the internal Pod map and use the controller runtime cache
# Description

Fixes: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1612

For large clusters it will be more efficient to use the controller-runtime cache directly instead of doing a list and then building our own cache on top of the controller-runtime cache.

## Type of change

*Please select one of the options below.*

- Breaking change (fix or feature that would cause existing functionality to not work as expected)

## Discussion

This is a breaking change as we extend the `PodLifecycleManager` interface.

## Testing

Adjusted unit tests and e2e tests will be running.

## Documentation

-

## Follow-up

Could be: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1619 to make sure we reduce the number of conversions and parsing in our operator.

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: b7a936c4b1d39f18a6114e0d20e3844cf4156476
* Duration 3:08:29
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/c2890a5a-af29-4fcb-a4d0-eced1c8595fc.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/c2890a5a-af29-4fcb-a4d0-eced1c8595fc/workspace.zip) zip file of the working directory (available for 30 days)

sbodagala: If there is already a controller run-time cache it makes sense to use that rather than building another index with the same functionality.
foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 053761cd0163229a8562be3ae606228799b3357a
* Duration 3:24:31
* Result: :white_check_mark: **SUCCEEDED**
* Error: `N/A`
* [Build Log](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/7623e489-6db3-4468-b44b-6a96afe6d3a6.gz) terminal output (available for 30 days)
* [Build Workspace](https://d1e0l78xbkh3xa.cloudfront.net/fdb-kubernetes-operator-pr/7623e489-6db3-4468-b44b-6a96afe6d3a6/workspace.zip) zip file of the working directory (available for 30 days)


Patch:
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index f046def0..ddb2e11e 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -361,6 +361,23 @@ func (processGroupStatus *ProcessGroupStatus) MarkForRemoval() {
 	processGroupStatus.RemovalTimestamp = &metav1.Time{Time: time.Now()}
 }
 
+// GetPodName returns the Pod name for the associated Process Group.
+func (processGroupStatus *ProcessGroupStatus) GetPodName(cluster *FoundationDBCluster) string {
+	var sb strings.Builder
+	sb.WriteString(cluster.Name)
+	sb.WriteString("-")
+	// The Pod name will always be in the format ${cluster}-${process-class}-${id}. The ID is currently not available
+	// in the processGroupStatus without doing any parsing, so we have to use the Process Group ID, which might contain
+	// a prefix, so we take the part after the prefix, which will be ${process-class}-${id}.
+	sanitizedProcessGroup := strings.ReplaceAll(string(processGroupStatus.ProcessGroupID), "_", "-")
+	sanitizedProcessClass := strings.ReplaceAll(string(processGroupStatus.ProcessClass), "_", "-")
+
+	idx := strings.Index(sanitizedProcessGroup, sanitizedProcessClass)
+	sb.WriteString(sanitizedProcessGroup[idx:])
+
+	return sb.String()
+}
+
 // NeedsReplacement checks if the ProcessGroupStatus has conditions so that it should be removed
 func (processGroupStatus *ProcessGroupStatus) NeedsReplacement(failureTime int, taintReplacementTime int) (bool, int64) {
 	var earliestFailureTime int64 = math.MaxInt64
diff --git a/controllers/add_pods.go b/controllers/add_pods.go
index 48cc2ca1..3a352326 100644
--- a/controllers/add_pods.go
+++ b/controllers/add_pods.go
@@ -56,18 +56,20 @@ func (a addPods) reconcile(ctx context.Context, r *FoundationDBClusterReconciler
 		return &requeue{curError: err}
 	}
 
-	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
-	if err != nil {
-		return &requeue{curError: err}
-	}
-
-	podMap := internal.CreatePodMap(cluster, pods)
-
 	for _, processGroup := range cluster.Status.ProcessGroups {
-		if _, podExists := podMap[processGroup.ProcessGroupID]; podExists {
+		_, err := r.PodLifecycleManager.GetPod(ctx, r, cluster, processGroup.GetPodName(cluster))
+		// If no error is returned the Pod exists
+		if err == nil {
 			continue
 		}
 
+		// Ignore the is not found error, as we are checking here if we should create Pods.
+		if err != nil {
+			if !k8serrors.IsNotFound(err) {
+				return &requeue{curError: err}
+			}
+		}
+
 		// If this process group is marked for removal, we normally don't want to spin it back up
 		// again. However, in a downscaling scenario, it could be that this is a storage node that
 		// is still draining its data onto another one. Therefore, we only want to leave it off
diff --git a/controllers/delete_pods_for_buggification.go b/controllers/delete_pods_for_buggification.go
index d11656aa..6e81cdc7 100644
--- a/controllers/delete_pods_for_buggification.go
+++ b/controllers/delete_pods_for_buggification.go
@@ -25,7 +25,6 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 
-	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
 	"github.com/go-logr/logr"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
@@ -38,12 +37,6 @@ type deletePodsForBuggification struct{}
 // reconcile runs the reconciler's work.
 func (d deletePodsForBuggification) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) *requeue {
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "deletePodsForBuggification")
-	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
-	if err != nil {
-		return &requeue{curError: err}
-	}
-
-	podMap := internal.CreatePodMap(cluster, pods)
 	crashLoopContainerProcessGroups := cluster.GetCrashLoopContainerProcessGroups()
 
 	noSchedulePods := make(map[fdbv1beta2.ProcessGroupID]fdbv1beta2.None, len(cluster.Spec.Buggify.NoSchedule))
@@ -59,8 +52,9 @@ func (d deletePodsForBuggification) reconcile(ctx context.Context, r *Foundation
 			continue
 		}
 
-		pod, ok := podMap[processGroup.ProcessGroupID]
-		if !ok || pod == nil {
+		pod, err := r.PodLifecycleManager.GetPod(ctx, r, cluster, processGroup.GetPodName(cluster))
+		// If a Pod is not found ignore it for now.
+		if err != nil {
 			logger.V(1).Info("Could not find Pod for process group ID",
 				"processGroupID", processGroup.ProcessGroupID)
 			continue
@@ -135,7 +129,7 @@ func (d deletePodsForBuggification) reconcile(ctx context.Context, r *Foundation
 	if len(updates) > 0 {
 		logger.Info("Deleting pods", "count", len(updates))
 		r.Recorder.Event(cluster, "Normal", "UpdatingPods", "Recreating pods for buggification")
-		err = r.PodLifecycleManager.UpdatePods(logr.NewContext(ctx, logger), r, cluster, updates, true)
+		err := r.PodLifecycleManager.UpdatePods(logr.NewContext(ctx, logger), r, cluster, updates, true)
 		if err != nil {
 			return &requeue{curError: err}
 		}
diff --git a/controllers/generate_initial_cluster_file.go b/controllers/generate_initial_cluster_file.go
index b94d0db2..b3987313 100644
--- a/controllers/generate_initial_cluster_file.go
+++ b/controllers/generate_initial_cluster_file.go
@@ -47,18 +47,12 @@ func (g generateInitialClusterFile) reconcile(ctx context.Context, r *Foundation
 	logger.Info("Generating initial cluster file")
 	r.Recorder.Event(cluster, corev1.EventTypeNormal, "ChangingCoordinators", "Choosing initial coordinators")
 
-	initialPods := []*corev1.Pod{}
-	candidateClasses := cluster.GetEligibleCandidateClasses()
-	for _, candidateClass := range candidateClasses {
-		pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, candidateClass, "")...)
-		if err != nil {
-			return &requeue{curError: err}
-		}
-		initialPods = append(initialPods, pods...)
+	processCounts, err := cluster.GetProcessCountsWithDefaults()
+	if err != nil {
+		return &requeue{curError: err}
 	}
 
-	podMap := internal.CreatePodMap(cluster, initialPods)
-	var pods = make([]*corev1.Pod, 0, len(initialPods))
+	var pods = make([]*corev1.Pod, 0, processCounts.Total())
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if processGroup.IsMarkedForRemoval() {
 			logger.V(1).Info("Ignore process group marked for removal",
@@ -66,9 +60,15 @@ func (g generateInitialClusterFile) reconcile(ctx context.Context, r *Foundation
 			continue
 		}
 
-		pod, ok := podMap[processGroup.ProcessGroupID]
-		if !ok {
-			logger.V(1).Info("Ignore process group with missing Pod",
+		// Ignore all process groups that are not eligible as a coordinator.
+		if !cluster.IsEligibleAsCandidate(processGroup.ProcessClass) {
+			continue
+		}
+
+		pod, err := r.PodLifecycleManager.GetPod(ctx, r, cluster, processGroup.GetPodName(cluster))
+		// If a Pod is not found ignore it for now.
+		if err != nil {
+			logger.V(1).Info("Could not find Pod for process group ID",
 				"processGroupID", processGroup.ProcessGroupID)
 			continue
 		}
diff --git a/controllers/remove_incompatible_processes.go b/controllers/remove_incompatible_processes.go
index cdf166da..90ba3ed5 100644
--- a/controllers/remove_incompatible_processes.go
+++ b/controllers/remove_incompatible_processes.go
@@ -29,7 +29,6 @@ import (
 	"github.com/go-logr/logr"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
-	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
 )
 
 // removeIncompatibleProcesses is a reconciler that will restart incompatible fdbserver processes, this can happen
@@ -81,18 +80,12 @@ func processIncompatibleProcesses(ctx context.Context, r *FoundationDBClusterRec
 
 	logger.Info("incompatible connections", "incompatibleConnections", status.Cluster.IncompatibleConnections)
 
-	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
-	if err != nil {
-		return err
-	}
-
-	podMap := internal.CreatePodMap(cluster, pods)
-
 	incompatibleConnections := parseIncompatibleConnections(logger, status)
 	incompatiblePods := make([]*corev1.Pod, 0, len(incompatibleConnections))
 	for _, processGroup := range cluster.Status.ProcessGroups {
-		pod, ok := podMap[processGroup.ProcessGroupID]
-		if !ok || pod == nil {
+		pod, err := r.PodLifecycleManager.GetPod(ctx, r, cluster, processGroup.GetPodName(cluster))
+		// If a Pod is not found ignore it for now.
+		if err != nil {
 			logger.V(1).Info("Could not find Pod for process group ID",
 				"processGroupID", processGroup.ProcessGroupID)
 			continue
diff --git a/controllers/replace_misconfigured_process_groups.go b/controllers/replace_misconfigured_process_groups.go
index fb1915ea..3e7fc3d7 100644
--- a/controllers/replace_misconfigured_process_groups.go
+++ b/controllers/replace_misconfigured_process_groups.go
@@ -39,18 +39,14 @@ type replaceMisconfiguredProcessGroups struct{}
 func (c replaceMisconfiguredProcessGroups) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) *requeue {
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "replaceMisconfiguredProcessGroups")
 
+	// TODO(johscheuer): Remove the pvc map an make direct calls.
 	pvcs := &corev1.PersistentVolumeClaimList{}
 	err := r.List(ctx, pvcs, internal.GetPodListOptions(cluster, "", "")...)
 	if err != nil {
 		return &requeue{curError: err}
 	}
 
-	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
-	if err != nil {
-		return &requeue{curError: err}
-	}
-
-	hasReplacements, err := replacements.ReplaceMisconfiguredProcessGroups(logger, cluster, internal.CreatePVCMap(cluster, pvcs), internal.CreatePodMap(cluster, pods))
+	hasReplacements, err := replacements.ReplaceMisconfiguredProcessGroups(ctx, r.PodLifecycleManager, r, logger, cluster, internal.CreatePVCMap(cluster, pvcs))
 	if err != nil {
 		return &requeue{curError: err}
 	}
diff --git a/controllers/update_labels.go b/controllers/update_labels.go
index 8c7b44fe..82fb6a88 100644
--- a/controllers/update_labels.go
+++ b/controllers/update_labels.go
@@ -36,14 +36,9 @@ type updateLabels struct{}
 // reconcile runs the reconciler's work.
 func (updateLabels) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbtypes.FoundationDBCluster) *requeue {
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "updateLabels")
-	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
-	if err != nil {
-		return &requeue{curError: err}
-	}
-	podMap := internal.CreatePodMap(cluster, pods)
-
+	// TODO(johscheuer): Remove the use of the pvc map and directly make a get request.
 	pvcs := &corev1.PersistentVolumeClaimList{}
-	err = r.List(ctx, pvcs, internal.GetPodListOptions(cluster, "", "")...)
+	err := r.List(ctx, pvcs, internal.GetPodListOptions(cluster, "", "")...)
 	if err != nil {
 		return &requeue{curError: err}
 	}
@@ -56,15 +51,15 @@ func (updateLabels) reconcile(ctx context.Context, r *FoundationDBClusterReconci
 			continue
 		}
 
-		pod, ok := podMap[processGroup.ProcessGroupID]
-		if ok {
+		pod, err := r.PodLifecycleManager.GetPod(ctx, r, cluster, processGroup.GetPodName(cluster))
+		if err == nil {
 			metadata := internal.GetPodMetadata(cluster, processGroup.ProcessClass, processGroup.ProcessGroupID, "")
 			if metadata.Annotations == nil {
 				metadata.Annotations = make(map[string]string, 1)
 			}
 
 			if !metadataCorrect(metadata, &pod.ObjectMeta) {
-				err = r.PodLifecycleManager.UpdateMetadata(ctx, r, cluster, pod)
+				err := r.PodLifecycleManager.UpdateMetadata(ctx, r, cluster, pod)
 				if err != nil {
 					return &requeue{curError: err}
 				}
diff --git a/controllers/update_pod_config.go b/controllers/update_pod_config.go
index 3919caed..f527eeda 100644
--- a/controllers/update_pod_config.go
+++ b/controllers/update_pod_config.go
@@ -45,13 +45,6 @@ func (updatePodConfig) reconcile(ctx context.Context, r *FoundationDBClusterReco
 		return &requeue{curError: err}
 	}
 
-	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
-	if err != nil {
-		return &requeue{curError: err}
-	}
-
-	podMap := internal.CreatePodMap(cluster, pods)
-
 	originalStatus := cluster.Status.DeepCopy()
 	allSynced := true
 	delayedRequeue := true
@@ -69,9 +62,10 @@ func (updatePodConfig) reconcile(ctx context.Context, r *FoundationDBClusterReco
 			continue
 		}
 
-		pod, ok := podMap[processGroup.ProcessGroupID]
-		if !ok || pod == nil {
-			curLogger.Info("Could not find Pod for process group")
+		pod, err := r.PodLifecycleManager.GetPod(ctx, r, cluster, processGroup.GetPodName(cluster))
+		// If a Pod is not found ignore it for now.
+		if err != nil {
+			logger.V(1).Info("Could not find Pod for process group ID")
 			continue
 		}
 
diff --git a/controllers/update_pods.go b/controllers/update_pods.go
index 1b9166e4..411b638e 100644
--- a/controllers/update_pods.go
+++ b/controllers/update_pods.go
@@ -42,12 +42,7 @@ type updatePods struct{}
 func (updatePods) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) *requeue {
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "updatePods")
 
-	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
-	if err != nil {
-		return &requeue{curError: err, delayedRequeue: true}
-	}
-
-	updates, err := getPodsToUpdate(logger, r, cluster, internal.CreatePodMap(cluster, pods))
+	updates, err := getPodsToUpdate(ctx, logger, r, cluster)
 	if err != nil {
 		return &requeue{curError: err, delay: podSchedulingDelayDuration, delayedRequeue: true}
 	}
@@ -84,7 +79,7 @@ func (updatePods) reconcile(ctx context.Context, r *FoundationDBClusterReconcile
 }
 
 // getPodsToUpdate returns a map of Zone to Pods mapping. The map has the fault domain as key and all Pods in that fault domain will be present as a slice of *corev1.Pod.
-func getPodsToUpdate(logger logr.Logger, reconciler *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, podMap map[fdbv1beta2.ProcessGroupID]*corev1.Pod) (map[string][]*corev1.Pod, error) {
+func getPodsToUpdate(ctx context.Context, logger logr.Logger, reconciler *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) (map[string][]*corev1.Pod, error) {
 	updates := make(map[string][]*corev1.Pod)
 
 	for _, processGroup := range cluster.Status.ProcessGroups {
@@ -106,8 +101,9 @@ func getPodsToUpdate(logger logr.Logger, reconciler *FoundationDBClusterReconcil
 			continue
 		}
 
-		pod, ok := podMap[processGroup.ProcessGroupID]
-		if !ok || pod == nil {
+		pod, err := reconciler.PodLifecycleManager.GetPod(ctx, reconciler, cluster, processGroup.GetPodName(cluster))
+		// If a Pod is not found ignore it for now.
+		if err != nil {
 			logger.V(1).Info("Could not find Pod for process group ID",
 				"processGroupID", processGroup.ProcessGroupID)
 			continue
diff --git a/controllers/update_sidecar_versions.go b/controllers/update_sidecar_versions.go
index fbf5db19..8f5060b9 100644
--- a/controllers/update_sidecar_versions.go
+++ b/controllers/update_sidecar_versions.go
@@ -45,13 +45,7 @@ func (updateSidecarVersions) reconcile(ctx context.Context, r *FoundationDBClust
 		return nil
 	}
 
-	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
-	if err != nil {
-		return &requeue{curError: err}
-	}
 	var upgraded int
-
-	podMap := internal.CreatePodMap(cluster, pods)
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if processGroup.GetConditionTime(fdbv1beta2.ResourcesTerminating) != nil {
 			logger.V(1).Info("Ignore process group that is stuck terminating",
@@ -59,8 +53,9 @@ func (updateSidecarVersions) reconcile(ctx context.Context, r *FoundationDBClust
 			continue
 		}
 
-		pod, ok := podMap[processGroup.ProcessGroupID]
-		if !ok || pod == nil {
+		pod, err := r.PodLifecycleManager.GetPod(ctx, r, cluster, processGroup.GetPodName(cluster))
+		// If a Pod is not found ignore it for now.
+		if err != nil {
 			logger.V(1).Info("Could not find Pod for process group ID",
 				"processGroupID", processGroup.ProcessGroupID)
 			continue
diff --git a/controllers/update_status.go b/controllers/update_status.go
index da39c8b7..826450db 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -166,12 +166,12 @@ func (updateStatus) reconcile(ctx context.Context, r *FoundationDBClusterReconci
 		}
 	}
 
-	pods, pvcs, err := refreshProcessGroupStatus(ctx, r, cluster, &status)
+	pvcs, err := refreshProcessGroupStatus(ctx, r, cluster, &status)
 	if err != nil {
 		return &requeue{curError: fmt.Errorf("update_status skipped due to error in refreshProcessGroupStatus: %w", err)}
 	}
 
-	status.ProcessGroups, err = validateProcessGroups(ctx, r, cluster, &status, processMap, configMap, pods, pvcs, logger)
+	status.ProcessGroups, err = validateProcessGroups(ctx, r, cluster, &status, processMap, configMap, pvcs, logger)
 	if err != nil {
 		return &requeue{curError: fmt.Errorf("update_status skipped due to error in validateProcessGroups: %w", err)}
 	}
@@ -408,7 +408,7 @@ func checkAndSetProcessStatus(r *FoundationDBClusterReconciler, cluster *fdbv1be
 }
 
 // Validate and set progressGroup's status
-func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBClusterStatus, processMap map[fdbv1beta2.ProcessGroupID][]fdbv1beta2.FoundationDBStatusProcessInfo, configMap *corev1.ConfigMap, pods []*corev1.Pod, pvcs *corev1.PersistentVolumeClaimList, logger logr.Logger) ([]*fdbv1beta2.ProcessGroupStatus, error) {
+func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBClusterStatus, processMap map[fdbv1beta2.ProcessGroupID][]fdbv1beta2.FoundationDBStatusProcessInfo, configMap *corev1.ConfigMap, pvcs *corev1.PersistentVolumeClaimList, logger logr.Logger) ([]*fdbv1beta2.ProcessGroupStatus, error) {
 	var err error
 	nodeMap := make(map[string]*corev1.Node)
 	processGroups := status.ProcessGroups
@@ -418,7 +418,6 @@ func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler
 		processGroupsWithoutExclusion[processGroupID] = fdbv1beta2.None{}
 	}
 
-	podMap := internal.CreatePodMap(cluster, pods)
 	pvcMap := internal.CreatePVCMap(cluster, pvcs)
 
 	disableTaintFeature := cluster.IsTaintFeatureDisabled()
@@ -427,11 +426,12 @@ func validateProcessGroups(ctx context.Context, r *FoundationDBClusterReconciler
 	}
 
 	for _, processGroup := range processGroups {
-		pod, podExists := podMap[processGroup.ProcessGroupID]
+		pod, podError := r.PodLifecycleManager.GetPod(ctx, r, cluster, processGroup.GetPodName(cluster))
+
 		// If the process group is not being removed and the Pod is not set we need to put it into
 		// the failing list.
 		isBeingRemoved := cluster.ProcessGroupIsBeingRemoved(processGroup.ProcessGroupID)
-		if !podExists {
+		if k8serrors.IsNotFound(podError) {
 			// Mark process groups as terminating if the pod has been deleted but other
 			// resources are stuck in terminating.
 			if isBeingRemoved {
@@ -753,7 +753,7 @@ func removeDuplicateConditions(status fdbv1beta2.FoundationDBClusterStatus) {
 	}
 }
 
-func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBClusterStatus) ([]*corev1.Pod, *corev1.PersistentVolumeClaimList, error) {
+func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, status *fdbv1beta2.FoundationDBClusterStatus) (*corev1.PersistentVolumeClaimList, error) {
 	status.ProcessGroups = make([]*fdbv1beta2.ProcessGroupStatus, 0, len(cluster.Status.ProcessGroups))
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if processGroup != nil && processGroup.ProcessGroupID != "" {
@@ -765,7 +765,7 @@ func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconc
 	// even if the process group is currently missing for some reasons.
 	pods, err := r.PodLifecycleManager.GetPods(ctx, r, cluster, internal.GetPodListOptions(cluster, "", "")...)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	for _, pod := range pods {
@@ -780,7 +780,7 @@ func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconc
 	pvcs := &corev1.PersistentVolumeClaimList{}
 	err = r.List(ctx, pvcs, internal.GetPodListOptions(cluster, "", "")...)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	for _, pvc := range pvcs.Items {
@@ -795,7 +795,7 @@ func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconc
 	services := &corev1.ServiceList{}
 	err = r.List(ctx, services, internal.GetPodListOptions(cluster, "", "")...)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	for _, service := range services.Items {
@@ -807,7 +807,7 @@ func refreshProcessGroupStatus(ctx context.Context, r *FoundationDBClusterReconc
 		status.ProcessGroups = append(status.ProcessGroups, fdbv1beta2.NewProcessGroupStatus(processGroupID, internal.ProcessClassFromLabels(cluster, service.Labels), nil))
 	}
 
-	return pods, pvcs, nil
+	return pvcs, nil
 }
 
 func getRunningVersion(versionMap map[string]int, fallback string) (string, error) {
diff --git a/internal/pod_helper.go b/internal/pod_helper.go
index bc2068bf..853eefe5 100644
--- a/internal/pod_helper.go
+++ b/internal/pod_helper.go
@@ -226,20 +226,6 @@ func GetSidecarImage(cluster *fdbv1beta2.FoundationDBCluster, pClass fdbv1beta2.
 	return GetImage(image, imageConfigs, cluster.Spec.Version, false)
 }
 
-// CreatePodMap creates a map with the process group ID as a key and the according Pod as a value
-func CreatePodMap(cluster *fdbv1beta2.FoundationDBCluster, pods []*corev1.Pod) map[fdbv1beta2.ProcessGroupID]*corev1.Pod {
-	podProcessGroupMap := make(map[fdbv1beta2.ProcessGroupID]*corev1.Pod, len(pods))
-	for _, pod := range pods {
-		processGroupID := GetProcessGroupIDFromMeta(cluster, pod.ObjectMeta)
-		if processGroupID == "" {
-			continue
-		}
-		podProcessGroupMap[processGroupID] = pod
-	}
-
-	return podProcessGroupMap
-}
-
 // ParseProcessGroupID extracts the components of an process group ID.
 func ParseProcessGroupID(id fdbv1beta2.ProcessGroupID) (fdbv1beta2.ProcessClass, int, error) {
 	result := processGroupIDRegex.FindStringSubmatch(string(id))
diff --git a/internal/replacements/replacements.go b/internal/replacements/replacements.go
index 733da729..8e0caa0d 100644
--- a/internal/replacements/replacements.go
+++ b/internal/replacements/replacements.go
@@ -21,7 +21,10 @@
 package replacements
 
 import (
+	"context"
 	"fmt"
+	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
@@ -33,7 +36,7 @@ import (
 )
 
 // ReplaceMisconfiguredProcessGroups checks if the cluster has any misconfigured process groups that must be replaced.
-func ReplaceMisconfiguredProcessGroups(log logr.Logger, cluster *fdbv1beta2.FoundationDBCluster, pvcMap map[fdbv1beta2.ProcessGroupID]corev1.PersistentVolumeClaim, podMap map[fdbv1beta2.ProcessGroupID]*corev1.Pod) (bool, error) {
+func ReplaceMisconfiguredProcessGroups(ctx context.Context, podManager podmanager.PodLifecycleManager, client client.Client, log logr.Logger, cluster *fdbv1beta2.FoundationDBCluster, pvcMap map[fdbv1beta2.ProcessGroupID]corev1.PersistentVolumeClaim) (bool, error) {
 	hasReplacements := false
 
 	maxReplacements := getMaxReplacements(cluster, cluster.GetMaxConcurrentReplacements())
@@ -47,16 +50,16 @@ func ReplaceMisconfiguredProcessGroups(log logr.Logger, cluster *fdbv1beta2.Foun
 			continue
 		}
 
+		// TODO(johscheuer): Fix how we fetch the pvc to make better use of the controller runtime cache.
 		pvc, hasPVC := pvcMap[processGroup.ProcessGroupID]
-		pod, hasPod := podMap[processGroup.ProcessGroupID]
-
+		pod, podErr := podManager.GetPod(ctx, client, cluster, processGroup.GetPodName(cluster))
 		if hasPVC {
 			needsPVCRemoval, err := processGroupNeedsRemovalForPVC(cluster, pvc, log)
 			if err != nil {
 				return hasReplacements, err
 			}
 
-			if needsPVCRemoval && hasPod {
+			if needsPVCRemoval && podErr == nil {
 				processGroup.MarkForRemoval()
 				hasReplacements = true
 				maxReplacements--
@@ -67,7 +70,7 @@ func ReplaceMisconfiguredProcessGroups(log logr.Logger, cluster *fdbv1beta2.Foun
 				"processGroupID", processGroup.ProcessGroupID)
 		}
 
-		if !hasPod || pod == nil {
+		if podErr != nil {
 			log.V(1).Info("Could not find Pod for process group ID",
 				"processGroupID", processGroup.ProcessGroupID)
 			continue
diff --git a/pkg/podmanager/pod_lifecycle_manager.go b/pkg/podmanager/pod_lifecycle_manager.go
index 87810da1..9b5b4cbe 100644
--- a/pkg/podmanager/pod_lifecycle_manager.go
+++ b/pkg/podmanager/pod_lifecycle_manager.go
@@ -34,13 +34,16 @@ import (
 // PodLifecycleManager provides an abstraction around Pod management to allow
 // using intermediary controllers that will manage the Pod lifecycle.
 type PodLifecycleManager interface {
-	// GetPods lists the Pods in the cluster
+	// GetPods lists the Pods in the cluster.
 	GetPods(context.Context, client.Client, *fdbv1beta2.FoundationDBCluster, ...client.ListOption) ([]*corev1.Pod, error)
 
-	// CreatePod creates a new Pod based on a pod definition
+	// GetPod returns the Pod for this cluster with the specified name.
+	GetPod(context.Context, client.Client, *fdbv1beta2.FoundationDBCluster, string) (*corev1.Pod, error)
+
+	// CreatePod creates a new Pod based on a pod definition.
 	CreatePod(context.Context, client.Client, *corev1.Pod) error
 
-	// DeletePod deletes a Pod
+	// DeletePod deletes a Pod.
 	DeletePod(context.Context, client.Client, *corev1.Pod) error
 
 	// CanDeletePods checks whether it is safe to delete pods.
@@ -70,8 +73,7 @@ type PodLifecycleManager interface {
 // that directly creates pods.
 type StandardPodLifecycleManager struct{}
 
-// GetPods returns a list of Pods for FDB pods that have been
-// created.
+// GetPods returns a list of Pods for FDB Pods that have been created.
 func (manager StandardPodLifecycleManager) GetPods(ctx context.Context, r client.Client, cluster *fdbv1beta2.FoundationDBCluster, options ...client.ListOption) ([]*corev1.Pod, error) {
 	pods := &corev1.PodList{}
 	err := r.List(ctx, pods, options...)
@@ -99,6 +101,14 @@ func (manager StandardPodLifecycleManager) GetPods(ctx context.Context, r client
 	return resPods, nil
 }
 
+// GetPod returns the Pod for this cluster with the specified name.
+func (manager StandardPodLifecycleManager) GetPod(ctx context.Context, r client.Client, cluster *fdbv1beta2.FoundationDBCluster, name string) (*corev1.Pod, error) {
+	pod := &corev1.Pod{}
+	err := r.Get(ctx, client.ObjectKey{Name: name, Namespace: cluster.Namespace}, pod)
+
+	return pod, err
+}
+
 // CreatePod creates a new Pod based on a Pod definition
 func (manager StandardPodLifecycleManager) CreatePod(ctx context.Context, r client.Client, pod *corev1.Pod) error {
 	logr.FromContextOrDiscard(ctx).V(1).Info("Creating pod", "name", pod.Name)


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index 41021076..4dcf55f4 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -5018,4 +5018,52 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 				},
 			}, true, false),
 	)
+
+	DescribeTable("when getting the Pod name for a Process group", func(cluster *FoundationDBCluster, processGroup *ProcessGroupStatus, expected string) {
+		Expect(processGroup.GetPodName(cluster)).To(Equal(expected))
+	}, Entry("when the process group has no prefix",
+		&FoundationDBCluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "testing-cluster",
+			},
+		},
+		&ProcessGroupStatus{
+			ProcessGroupID: "storage-1",
+			ProcessClass:   ProcessClassStorage,
+		},
+		"testing-cluster-storage-1"),
+		Entry("when the process group has a prefix",
+			&FoundationDBCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testing-cluster",
+				},
+			},
+			&ProcessGroupStatus{
+				ProcessGroupID: "this-is-my-fancy-prefix-storage-1",
+				ProcessClass:   ProcessClassStorage,
+			},
+			"testing-cluster-storage-1"),
+		Entry("when the process group has no prefix and the process class has an underscore",
+			&FoundationDBCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testing-cluster",
+				},
+			},
+			&ProcessGroupStatus{
+				ProcessGroupID: "cluster-controller-1",
+				ProcessClass:   ProcessClassClusterController,
+			},
+			"testing-cluster-cluster-controller-1"),
+		Entry("when the process group has a prefix and the process class has an underscore",
+			&FoundationDBCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testing-cluster",
+				},
+			},
+			&ProcessGroupStatus{
+				ProcessGroupID: "this-is-my-fancy-prefix-cluster-controller-1",
+				ProcessClass:   ProcessClassClusterController,
+			},
+			"testing-cluster-cluster-controller-1"),
+	)
 })
diff --git a/controllers/replace_failed_process_groups_test.go b/controllers/replace_failed_process_groups_test.go
index d671e107..3d5cc56e 100644
--- a/controllers/replace_failed_process_groups_test.go
+++ b/controllers/replace_failed_process_groups_test.go
@@ -74,7 +74,7 @@ var _ = Describe("replace_failed_process_groups", func() {
 		taintKeyStarDuration := int64(20)
 		taintKeyMaintenance := "foundationdb.org/maintenance"
 		taintKeyMaintenanceDuration := int64(10)
-		var allPods []*corev1.Pod
+
 		var allPvcs *corev1.PersistentVolumeClaimList
 		var podOnTaintedNode *corev1.Pod
 		var targetPodProcessGroupID fdbv1beta2.ProcessGroupID
@@ -117,9 +117,6 @@ var _ = Describe("replace_failed_process_groups", func() {
 				processMap[fdbv1beta2.ProcessGroupID(processID)] = append(processMap[fdbv1beta2.ProcessGroupID(processID)], process)
 			}
 
-			allPods, err = clusterReconciler.PodLifecycleManager.GetPods(ctx.TODO(), clusterReconciler, cluster, internal.GetPodListOptions(cluster, "", "")...)
-			Expect(err).NotTo(HaveOccurred())
-
 			allPvcs = &corev1.PersistentVolumeClaimList{}
 			err = clusterReconciler.List(ctx.TODO(), allPvcs, internal.GetPodListOptions(cluster, "", "")...)
 			Expect(err).NotTo(HaveOccurred())
@@ -139,7 +136,7 @@ var _ = Describe("replace_failed_process_groups", func() {
 			targetPodProcessGroupID = internal.GetProcessGroupIDFromMeta(cluster, podOnTaintedNode.ObjectMeta)
 
 			// Call validateProcessGroups to set processGroupStatus to tainted condition
-			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(processGroupsStatus)).To(BeNumerically(">", 4))
 			processGroup := processGroupsStatus[len(processGroupsStatus)-4]
@@ -159,7 +156,7 @@ var _ = Describe("replace_failed_process_groups", func() {
 			Expect(k8sClient.Update(ctx.TODO(), node)).NotTo(HaveOccurred())
 			log.Info("Taint node", "Node name", podOnTaintedNode.Name, "Node taints", node.Spec.Taints)
 
-			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(processGroupsStatus)).To(BeNumerically(">", 4))
 			targetProcessGroupStatus := fdbv1beta2.FindProcessGroupByID(cluster.Status.ProcessGroups, targetPodProcessGroupID)
@@ -183,7 +180,7 @@ var _ = Describe("replace_failed_process_groups", func() {
 			Expect(k8sClient.Update(ctx.TODO(), node)).NotTo(HaveOccurred())
 			log.Info("Taint node", "Node name", podOnTaintedNode.Name, "Node taints", node.Spec.Taints)
 
-			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(processGroupsStatus)).To(BeNumerically(">", 4))
 			targetProcessGroupStatus := fdbv1beta2.FindProcessGroupByID(cluster.Status.ProcessGroups, targetPodProcessGroupID)
@@ -194,7 +191,7 @@ var _ = Describe("replace_failed_process_groups", func() {
 			node.Spec.Taints = []corev1.Taint{}
 			err = k8sClient.Update(ctx.TODO(), node)
 			Expect(err).NotTo(HaveOccurred())
-			processGroupsStatus, err = validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+			processGroupsStatus, err = validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(processGroupsStatus)).To(BeNumerically(">", 4))
 			targetProcessGroupStatus = fdbv1beta2.FindProcessGroupByID(cluster.Status.ProcessGroups, targetPodProcessGroupID)
@@ -217,7 +214,7 @@ var _ = Describe("replace_failed_process_groups", func() {
 			log.Info("Taint node", "Node name", podOnTaintedNode.Name, "Node taints", node.Spec.Taints, "TaintTime", node.Spec.Taints[0].TimeAdded.Time, "Now", time.Now())
 			Expect(k8sClient.Update(ctx.TODO(), node)).NotTo(HaveOccurred())
 
-			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(processGroupsStatus)).To(BeNumerically(">", 4))
 			targetProcessGroupStatus := fdbv1beta2.FindProcessGroupByID(cluster.Status.ProcessGroups, targetPodProcessGroupID)
@@ -249,7 +246,7 @@ var _ = Describe("replace_failed_process_groups", func() {
 			log.Info("Taint node", "Node name", podOnTaintedNode.Name, "Node taints", node.Spec.Taints, "TaintTime", node.Spec.Taints[0].TimeAdded.Time, "Now", time.Now())
 			Expect(k8sClient.Update(ctx.TODO(), node)).NotTo(HaveOccurred())
 
-			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(processGroupsStatus)).To(BeNumerically(">", 4))
 			targetProcessGroupStatus := fdbv1beta2.FindProcessGroupByID(cluster.Status.ProcessGroups, targetPodProcessGroupID)
@@ -287,7 +284,7 @@ var _ = Describe("replace_failed_process_groups", func() {
 			}
 			Expect(tainted).To(Equal(int64(0)))
 
-			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+			processGroupsStatus, err := validateProcessGroups(ctx.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(processGroupsStatus)).To(BeNumerically(">", 4))
 			targetProcessGroupStatus := fdbv1beta2.FindProcessGroupByID(cluster.Status.ProcessGroups, targetPodProcessGroupID)
@@ -565,7 +562,11 @@ var _ = Describe("replace_failed_process_groups", func() {
 		When("multiple nodes are tainted", func() {
 			var taintedNodes []*corev1.Node
 			var setValidTaint bool
+
 			BeforeEach(func() {
+				allPods, err := clusterReconciler.PodLifecycleManager.GetPods(ctx.TODO(), clusterReconciler, cluster, internal.GetPodListOptions(cluster, "", "")...)
+				Expect(err).NotTo(HaveOccurred())
+
 				concurrentTaints := 2
 				Expect(len(allPods)).To(BeNumerically(">", concurrentTaints))
 				taintedNodesIndex := map[int]struct{}{}
@@ -628,7 +629,6 @@ var _ = Describe("replace_failed_process_groups", func() {
 				Expect(getRemovedProcessGroupIDs(cluster)).To(Equal([]fdbv1beta2.ProcessGroupID{}))
 			})
 		})
-
 	})
 
 	Context("with no missing processes", func() {
diff --git a/controllers/suite_test.go b/controllers/suite_test.go
index 3d31598f..982396ae 100644
--- a/controllers/suite_test.go
+++ b/controllers/suite_test.go
@@ -70,10 +70,8 @@ func TestAPIs(t *testing.T) {
 var _ = BeforeSuite(func() {
 	logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)))
 
-	err := scheme.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-	err = fdbv1beta2.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
+	Expect(scheme.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
+	Expect(fdbv1beta2.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
 
 	// +kubebuilder:scaffold:scheme
 	k8sClient = mockclient.NewMockClient(scheme.Scheme)
diff --git a/controllers/update_pods_test.go b/controllers/update_pods_test.go
index 983b8ed9..76649203 100644
--- a/controllers/update_pods_test.go
+++ b/controllers/update_pods_test.go
@@ -215,10 +215,8 @@ var _ = Describe("update_pods", func() {
 		})
 
 		JustBeforeEach(func() {
-			pods, err := clusterReconciler.PodLifecycleManager.GetPods(context.TODO(), k8sClient, cluster, internal.GetPodListOptions(cluster, "", "")...)
-			Expect(err).NotTo(HaveOccurred())
-
-			updates, err = getPodsToUpdate(log, clusterReconciler, cluster, internal.CreatePodMap(cluster, pods))
+			var err error
+			updates, err = getPodsToUpdate(context.Background(), log, clusterReconciler, cluster)
 			if !expectedError {
 				Expect(err).NotTo(HaveOccurred())
 			} else {
diff --git a/controllers/update_status_test.go b/controllers/update_status_test.go
index 100b6b8f..f6a4f3c9 100644
--- a/controllers/update_status_test.go
+++ b/controllers/update_status_test.go
@@ -22,6 +22,7 @@ package controllers
 
 import (
 	"context"
+	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -64,7 +65,6 @@ var _ = Describe("update_status", func() {
 
 		BeforeEach(func() {
 			var pods []*corev1.Pod
-			var allPods []*corev1.Pod
 			nodeMap = make(map[string]*corev1.Node)
 			cluster = internal.CreateDefaultCluster()
 			err = setupClusterForTest(cluster)
@@ -90,9 +90,6 @@ var _ = Describe("update_status", func() {
 				ObjectMeta: metav1.ObjectMeta{Name: pod.Spec.NodeName},
 			}
 
-			allPods, err = clusterReconciler.PodLifecycleManager.GetPods(context.TODO(), clusterReconciler, cluster, internal.GetPodListOptions(cluster, "", "")...)
-			Expect(err).NotTo(HaveOccurred())
-
 			allPvcs := &corev1.PersistentVolumeClaimList{}
 			err = clusterReconciler.List(context.TODO(), allPvcs, internal.GetPodListOptions(cluster, "", "")...)
 			Expect(err).NotTo(HaveOccurred())
@@ -105,8 +102,7 @@ var _ = Describe("update_status", func() {
 			}
 			log.Info("Target processGroupStatus Info", "ProcessGroupID", processGroupStatus.ProcessGroupID,
 				"Conditions size", len(processGroupStatus.ProcessGroupConditions),
-				"Conditions", processGroupStatus.ProcessGroupConditions,
-				"AllPods size", len(allPods))
+				"Conditions", processGroupStatus.ProcessGroupConditions)
 		})
 
 		When("process group's node is tainted", func() {
@@ -295,7 +291,6 @@ var _ = Describe("update_status", func() {
 		var storagePod *corev1.Pod
 		var processMap map[fdbv1beta2.ProcessGroupID][]fdbv1beta2.FoundationDBStatusProcessInfo
 		var err error
-		var allPods []*corev1.Pod
 		var allPvcs *corev1.PersistentVolumeClaimList
 		var nodeMap map[string]*corev1.Node
 
@@ -333,9 +328,6 @@ var _ = Describe("update_status", func() {
 				processMap[fdbv1beta2.ProcessGroupID(processID)] = append(processMap[fdbv1beta2.ProcessGroupID(processID)], process)
 			}
 
-			allPods, err = clusterReconciler.PodLifecycleManager.GetPods(context.TODO(), clusterReconciler, cluster, internal.GetPodListOptions(cluster, "", "")...)
-			Expect(err).NotTo(HaveOccurred())
-
 			allPvcs = &corev1.PersistentVolumeClaimList{}
 			err = clusterReconciler.List(context.TODO(), allPvcs, internal.GetPodListOptions(cluster, "", "")...)
 			Expect(err).NotTo(HaveOccurred())
@@ -354,7 +346,7 @@ var _ = Describe("update_status", func() {
 
 		When("a process group is fine", func() {
 			It("should not get any condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(processGroupStatus)).To(BeNumerically(">", 4))
 				processGroup := processGroupStatus[len(processGroupStatus)-4]
@@ -369,7 +361,9 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				dummyPod := &corev1.Pod{}
+				Expect(k8sClient.Get(context.TODO(), ctrlClient.ObjectKeyFromObject(storagePod), dummyPod)).To(HaveOccurred())
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				missingProcesses := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.MissingPod, false)
@@ -388,7 +382,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				incorrectProcesses := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.IncorrectCommandLine, false)
@@ -406,7 +400,7 @@ var _ = Describe("update_status", func() {
 				})
 
 				It("should get a condition assigned", func() {
-					processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+					processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 					Expect(err).NotTo(HaveOccurred())
 
 					incorrectProcesses := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.IncorrectCommandLine, false)
@@ -427,7 +421,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				missingProcesses := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.MissingProcesses, false)
@@ -448,7 +442,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				incorrectPods := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.IncorrectPodSpec, false)
@@ -472,9 +466,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				// Ensure that we have the same number of Pods and processes
-				Expect(allPods).To(HaveLen(len(processMap)))
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				missingProcesses := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.PodFailing, false)
@@ -495,7 +487,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				failingPods := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.PodFailing, false)
@@ -516,7 +508,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should get a condition assigned", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				failingPods := fdbv1beta2.FilterByCondition(processGroupStatus, fdbv1beta2.PodFailing, false)
@@ -541,7 +533,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should mark the process group for removal", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				removalCount := 0
@@ -572,7 +564,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should be mark the process group for removal without exclusion", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				removalCount := 0
@@ -600,19 +592,10 @@ var _ = Describe("update_status", func() {
 				unreachableProcessGroup = podmanager.GetProcessGroupID(cluster, storagePod)
 				unreachablePod.Annotations[internal.MockUnreachableAnnotation] = "banana"
 				Expect(k8sClient.Update(context.TODO(), unreachablePod)).NotTo(HaveOccurred())
-
-				// Update the Pod in our list, since we fetch the list before we set the annotation
-				for idx, pod := range allPods {
-					if pod.Name != unreachablePod.Name {
-						continue
-					}
-
-					allPods[idx] = unreachablePod
-				}
 			})
 
 			It("should mark the process group as unreachable", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				unreachableCount := 0
@@ -636,7 +619,7 @@ var _ = Describe("update_status", func() {
 				})
 
 				It("should remove the condition", func() {
-					processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+					processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 					Expect(err).NotTo(HaveOccurred())
 
 					unreachableCount := 0
@@ -664,7 +647,7 @@ var _ = Describe("update_status", func() {
 			})
 
 			It("should mark the process group as Pod pending", func() {
-				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPods, allPvcs, logger)
+				processGroupStatus, err := validateProcessGroups(context.TODO(), clusterReconciler, cluster, &cluster.Status, processMap, configMap, allPvcs, logger)
 				Expect(err).NotTo(HaveOccurred())
 
 				pendingCount := 0
diff --git a/internal/replacements/replacements_test.go b/internal/replacements/replacements_test.go
index 02aefcf3..d1658fbe 100644
--- a/internal/replacements/replacements_test.go
+++ b/internal/replacements/replacements_test.go
@@ -21,7 +21,10 @@
 package replacements
 
 import (
+	"context"
 	"fmt"
+	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager"
+	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -602,11 +605,9 @@ var _ = Describe("replace_misconfigured_pods", func() {
 
 	When("using MaxConcurrentMisconfiguredReplacements", func() {
 		var pvcMap map[fdbv1beta2.ProcessGroupID]corev1.PersistentVolumeClaim
-		var podMap map[fdbv1beta2.ProcessGroupID]*corev1.Pod
 
 		BeforeEach(func() {
 			pvcMap = map[fdbv1beta2.ProcessGroupID]corev1.PersistentVolumeClaim{}
-			podMap = map[fdbv1beta2.ProcessGroupID]*corev1.Pod{}
 
 			for i := 0; i < 10; i++ {
 				_, id := internal.GetProcessGroupID(cluster, fdbv1beta2.ProcessClassStorage, i)
@@ -615,7 +616,7 @@ var _ = Describe("replace_misconfigured_pods", func() {
 				pvcMap[id] = *newPVC
 				newPod, err := internal.GetPod(cluster, fdbv1beta2.ProcessClassStorage, i)
 				Expect(err).NotTo(HaveOccurred())
-				podMap[id] = newPod
+				Expect(k8sClient.Create(context.Background(), newPod)).NotTo(HaveOccurred())
 				// Populate process groups
 				cluster.Status.ProcessGroups = append(cluster.Status.ProcessGroups, fdbv1beta2.NewProcessGroupStatus(id, fdbv1beta2.ProcessClassStorage, nil))
 			}
@@ -627,7 +628,7 @@ var _ = Describe("replace_misconfigured_pods", func() {
 				pvcMap[id] = *newPVC
 				newPod, err := internal.GetPod(cluster, fdbv1beta2.ProcessClassTransaction, i)
 				Expect(err).NotTo(HaveOccurred())
-				podMap[id] = newPod
+				Expect(k8sClient.Create(context.Background(), newPod)).NotTo(HaveOccurred())
 				// Populate process groups
 				cluster.Status.ProcessGroups = append(cluster.Status.ProcessGroups, fdbv1beta2.NewProcessGroupStatus(id, fdbv1beta2.ProcessClassTransaction, nil))
 			}
@@ -644,7 +645,7 @@ var _ = Describe("replace_misconfigured_pods", func() {
 			})
 
 			It("should not have a replacements", func() {
-				hasReplacement, err := ReplaceMisconfiguredProcessGroups(log, cluster, pvcMap, podMap)
+				hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(hasReplacement).To(BeFalse())
 
@@ -667,7 +668,7 @@ var _ = Describe("replace_misconfigured_pods", func() {
 			})
 
 			It("should have two replacements", func() {
-				hasReplacement, err := ReplaceMisconfiguredProcessGroups(log, cluster, pvcMap, podMap)
+				hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(hasReplacement).To(BeTrue())
 
@@ -686,7 +687,7 @@ var _ = Describe("replace_misconfigured_pods", func() {
 
 		When("Setting is unset", func() {
 			It("should replace all process groups", func() {
-				hasReplacement, err := ReplaceMisconfiguredProcessGroups(log, cluster, pvcMap, podMap)
+				hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(hasReplacement).To(BeTrue())
 
@@ -710,9 +711,11 @@ var _ = Describe("replace_misconfigured_pods", func() {
 
 			When("the process is a storage process", func() {
 				BeforeEach(func() {
-					_, id := internal.GetProcessGroupID(cluster, fdbv1beta2.ProcessClassStorage, 0)
-					pod := podMap[id]
-					spec := pod.Spec.DeepCopy()
+					podName, _ := internal.GetProcessGroupID(cluster, fdbv1beta2.ProcessClassStorage, 0)
+					currentPod := &corev1.Pod{}
+					Expect(k8sClient.Get(context.Background(), ctrlClient.ObjectKey{Name: podName, Namespace: cluster.Namespace}, currentPod)).NotTo(HaveOccurred())
+
+					spec := currentPod.Spec.DeepCopy()
 					var cIdx int
 					for idx, con := range spec.Containers {
 						if con.Name != fdbv1beta2.MainContainerName {
@@ -724,11 +727,12 @@ var _ = Describe("replace_misconfigured_pods", func() {
 					}
 
 					spec.Containers[cIdx].Image = "banana"
-					pod.Spec = *spec
+					currentPod.Spec = *spec
+					Expect(k8sClient.Update(context.Background(), currentPod)).NotTo(HaveOccurred())
 				})
 
 				It("should not have any replacements", func() {
-					hasReplacement, err := ReplaceMisconfiguredProcessGroups(log, cluster, pvcMap, podMap)
+					hasReplacement, err := ReplaceMisconfiguredProcessGroups(context.Background(), podmanager.StandardPodLifecycleManager{}, k8sClient, log, cluster, pvcMap)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(hasReplacement).To(BeFalse())
 
diff --git a/internal/replacements/suite_test.go b/internal/replacements/suite_test.go
index aeed1b0c..6558cd68 100644
--- a/internal/replacements/suite_test.go
+++ b/internal/replacements/suite_test.go
@@ -1,6 +1,11 @@
 package replacements
 
 import (
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	mockclient "github.com/FoundationDB/fdb-kubernetes-operator/mock-kubernetes-client/client"
+	"k8s.io/client-go/kubernetes/scheme"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -11,3 +16,17 @@ func TestCmd(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Replacements Suite")
 }
+
+var k8sClient *mockclient.MockClient
+
+var _ = BeforeSuite(func() {
+	logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)))
+
+	Expect(scheme.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
+	Expect(fdbv1beta2.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
+	k8sClient = mockclient.NewMockClient(scheme.Scheme)
+})
+
+var _ = AfterEach(func() {
+	k8sClient.Clear()
+})