Repository: cloudnative-pg/cloudnative-pg

Issue:
[Feature]: Wait for init containers to complete before pod deletion in restored clusters handling
### Is there an existing issue already for this feature request/idea?

- [x] I have searched for an existing issue, and could not find anything. I believe this is a new feature request to be evaluated.

### What problem is this feature going to solve? Why should it be added?

Currently, during restored cluster handling, pods may be deleted even if some of their init containers (that do not have restartPolicy set to Always) are still running. This behavior can cause issues with unfinished initialization processes carrying out the actual data copy.

### Describe the solution you'd like

Enhance the logic in the ensureClusterRestoreCanStart function in internal/controller/cluster_restore.go to wait for the completion of any init container that does not have a restart policy of Always before proceeding with pod deletion.

### Describe alternatives you've considered

Alternative solutions considered:
- Not checking the state of init containers and deleting pods immediately (current behavior, which is unsafe)
- Deleting pods after a fixed wait time (not as robust or specific to init container states)

The proposed solution actively inspects pod status and init container policies, providing correct and explicit handling.

### Additional context

Related code: internal/controller/cluster_restore.go, specifically the ensureClusterRestoreCanStart function, where this logic can be inserted. The new logic should check pods managed for restoration and ensure no init container (without restartPolicy=Always) is running before deleting the pod.

### Backport?

No

### Are you willing to actively contribute to this feature?

Yes

### Code of Conduct

- [x] I agree to follow this project's Code of Conduct

Discussion:

Solution patch:
diff --git a/internal/controller/cluster_restore.go b/internal/controller/cluster_restore.go
index 7205e508..f53a8df3 100644
--- a/internal/controller/cluster_restore.go
+++ b/internal/controller/cluster_restore.go
@@ -23,6 +23,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/cloudnative-pg/machinery/pkg/log"
 	corev1 "k8s.io/api/core/v1"
@@ -189,7 +190,7 @@ func ensureClusterRestoreCanStart(
 	c client.Client,
 	cluster *apiv1.Cluster,
 ) (*ctrl.Result, error) {
-	return nil, nil
+	return ensureInitContainersAreCompleted(ctx, c, cluster)
 }
 
 func ensureClusterIsNotFenced(
@@ -349,3 +350,123 @@ func restoreOrphanPVCs(
 
 	return nil
 }
+
+func ensureInitContainersAreCompleted(
+	ctx context.Context,
+	cli client.Client,
+	cluster *apiv1.Cluster,
+) (*ctrl.Result, error) {
+	contextLogger := log.FromContext(ctx).WithValues("step", "ensure_init_containers_are_completed")
+
+	var podList corev1.PodList
+	if err := cli.List(
+		ctx,
+		&podList,
+		client.InNamespace(cluster.Namespace),
+		client.MatchingLabels{
+			utils.ClusterLabelName: cluster.Name,
+			utils.PodRoleLabelName: string(utils.PodRoleInstance),
+		},
+	); err != nil {
+		return nil, err
+	}
+
+	// Get all pods with non-sidecar init containers
+	podsWithInitContainers := getPodsWithNonSidecarInitContainers(podList)
+	if len(podsWithInitContainers) == 0 {
+		return nil, nil
+	}
+
+	// Check all pods and their init containers
+	for _, pod := range podsWithInitContainers {
+		// Check all non-sidecar init containers in this pod
+		nonSidecarStatuses := getNonSidecarInitContainerStatuses(
+			pod.Status.InitContainerStatuses,
+			pod.Spec.InitContainers,
+		)
+
+		if len(nonSidecarStatuses) == 0 {
+			contextLogger.Info("waiting for init containers to start", "podName", pod.Name)
+			return &ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+		}
+
+		// Check if any non-sidecar init container is still running or hasn't started
+		for _, status := range nonSidecarStatuses {
+			if status.State.Terminated == nil {
+				contextLogger.Info("init container running, waiting for completion",
+					"podName", pod.Name,
+					"initContainerName", status.Name)
+				return &ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+			}
+		}
+
+		// Check if any non-sidecar init container failed
+		for _, status := range nonSidecarStatuses {
+			if status.State.Terminated.ExitCode != 0 {
+				contextLogger.Info("init container failed",
+					"podName", pod.Name,
+					"initContainerName", status.Name,
+					"exitCode", status.State.Terminated.ExitCode,
+					"reason", status.State.Terminated.Reason,
+					"message", status.State.Terminated.Message)
+				return &ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+			}
+		}
+	}
+
+	contextLogger.Info("all init containers in all pods completed, proceeding to orphan pods deletion")
+	return nil, nil
+}
+
+func getPodsWithNonSidecarInitContainers(podList corev1.PodList) []*corev1.Pod {
+	var podsWithInitContainers []*corev1.Pod
+	for idx := range podList.Items {
+		pod := podList.Items[idx]
+		if hasNonSidecarInitContainers(&pod) {
+			podsWithInitContainers = append(podsWithInitContainers, &pod)
+		}
+	}
+	return podsWithInitContainers
+}
+
+func hasNonSidecarInitContainers(pod *corev1.Pod) bool {
+	for _, initContainer := range pod.Spec.InitContainers {
+		if isSidecarInitContainer(&initContainer) {
+			continue
+		}
+		return true
+	}
+
+	return false
+}
+
+func isSidecarInitContainer(initContainer *corev1.Container) bool {
+	return initContainer.RestartPolicy != nil && *initContainer.RestartPolicy == corev1.ContainerRestartPolicyAlways
+}
+
+func getNonSidecarInitContainerStatuses(
+	statuses []corev1.ContainerStatus,
+	initContainers []corev1.Container,
+) []corev1.ContainerStatus {
+	// Build a map of init container names to their specs for quick lookup
+	initContainerSpecMap := make(map[string]*corev1.Container)
+	for i := range initContainers {
+		initContainerSpecMap[initContainers[i].Name] = &initContainers[i]
+	}
+
+	// Collect all non-sidecar init container statuses
+	nonSidecarStatuses := make([]corev1.ContainerStatus, 0, len(statuses))
+	for i := range statuses {
+		status := statuses[i]
+		initContainerSpec, exists := initContainerSpecMap[status.Name]
+		if !exists {
+			continue
+		}
+
+		if !isSidecarInitContainer(initContainerSpec) {
+			nonSidecarStatuses = append(nonSidecarStatuses, status)
+		}
+	}
+
+	return nonSidecarStatuses
+}



Test patch:
diff --git a/internal/controller/cluster_restore_test.go b/internal/controller/cluster_restore_test.go
index fc67aa18..bd1ece79 100644
--- a/internal/controller/cluster_restore_test.go
+++ b/internal/controller/cluster_restore_test.go
@@ -21,6 +21,7 @@ package controller
 
 import (
 	"context"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
@@ -500,3 +501,751 @@ var _ = Describe("ensureOrphanServicesAreNotPresent", func() {
 		})
 	})
 })
+
+var _ = Describe("ensureInitContainersAreCompleted", func() {
+	var (
+		mockCli k8client.Client
+		cluster *apiv1.Cluster
+		pod     *corev1.Pod
+	)
+
+	BeforeEach(func() {
+		cluster = &apiv1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "test-cluster",
+				Namespace: "default",
+			},
+		}
+	})
+
+	Context("when no pods with init containers exist", func() {
+		BeforeEach(func() {
+			mockCli = fake.NewClientBuilder().
+				WithScheme(k8scheme.BuildWithAllKnownScheme()).
+				WithObjects(cluster).
+				Build()
+		})
+
+		It("should return nil without error", func(ctx SpecContext) {
+			res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+		})
+	})
+
+	Context("when a pod with init containers exists", func() {
+		BeforeEach(func() {
+			pod = &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-pod",
+					Namespace: "default",
+					Labels: map[string]string{
+						utils.ClusterLabelName: cluster.Name,
+						utils.PodRoleLabelName: string(utils.PodRoleInstance),
+					},
+				},
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{
+							Name: "restore-init",
+						},
+					},
+				},
+			}
+			mockCli = fake.NewClientBuilder().
+				WithScheme(k8scheme.BuildWithAllKnownScheme()).
+				WithObjects(cluster, pod).
+				Build()
+		})
+
+		Context("when init containers have not started", func() {
+			It("should requeue with 5 second delay", func(ctx SpecContext) {
+				res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(res).ToNot(BeNil())
+				Expect(res.RequeueAfter).To(Equal(5 * time.Second))
+			})
+		})
+
+		Context("when init containers are still running", func() {
+			BeforeEach(func() {
+				pod.Status.InitContainerStatuses = []corev1.ContainerStatus{
+					{
+						Name: "restore-init",
+						State: corev1.ContainerState{
+							Running: &corev1.ContainerStateRunning{},
+						},
+					},
+				}
+				mockCli = fake.NewClientBuilder().
+					WithScheme(k8scheme.BuildWithAllKnownScheme()).
+					WithObjects(cluster, pod).
+					Build()
+			})
+
+			It("should requeue with 5 second delay", func(ctx SpecContext) {
+				res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(res).ToNot(BeNil())
+				Expect(res.RequeueAfter).To(Equal(5 * time.Second))
+			})
+		})
+
+		Context("when init containers completed successfully", func() {
+			BeforeEach(func() {
+				pod.Status.InitContainerStatuses = []corev1.ContainerStatus{
+					{
+						Name: "restore-init",
+						State: corev1.ContainerState{
+							Terminated: &corev1.ContainerStateTerminated{
+								ExitCode: 0,
+								Reason:   "Completed",
+							},
+						},
+					},
+				}
+				mockCli = fake.NewClientBuilder().
+					WithScheme(k8scheme.BuildWithAllKnownScheme()).
+					WithObjects(cluster, pod).
+					Build()
+			})
+
+			It("should return nil without requeue", func(ctx SpecContext) {
+				res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(res).To(BeNil())
+			})
+		})
+
+		Context("when init containers failed", func() {
+			BeforeEach(func() {
+				pod.Status.InitContainerStatuses = []corev1.ContainerStatus{
+					{
+						Name: "restore-init",
+						State: corev1.ContainerState{
+							Terminated: &corev1.ContainerStateTerminated{
+								ExitCode: 1,
+								Reason:   "Error",
+								Message:  "restore failed",
+							},
+						},
+					},
+				}
+				mockCli = fake.NewClientBuilder().
+					WithScheme(k8scheme.BuildWithAllKnownScheme()).
+					WithObjects(cluster, pod).
+					Build()
+			})
+
+			It("should requeue with 10 second delay", func(ctx SpecContext) {
+				res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(res).ToNot(BeNil())
+				Expect(res.RequeueAfter).To(Equal(10 * time.Second))
+			})
+		})
+
+		Context("when multiple non-sidecar init containers exist", func() {
+			BeforeEach(func() {
+				pod.Spec.InitContainers = []corev1.Container{
+					{
+						Name: "init-1",
+					},
+					{
+						Name: "init-2",
+					},
+				}
+				pod.Status.InitContainerStatuses = []corev1.ContainerStatus{
+					{
+						Name: "init-1",
+						State: corev1.ContainerState{
+							Terminated: &corev1.ContainerStateTerminated{
+								ExitCode: 0,
+								Reason:   "Completed",
+							},
+						},
+					},
+					{
+						Name: "init-2",
+						State: corev1.ContainerState{
+							Terminated: &corev1.ContainerStateTerminated{
+								ExitCode: 0,
+								Reason:   "Completed",
+							},
+						},
+					},
+				}
+				mockCli = fake.NewClientBuilder().
+					WithScheme(k8scheme.BuildWithAllKnownScheme()).
+					WithObjects(cluster, pod).
+					Build()
+			})
+
+			It("should wait for all of them to complete", func(ctx SpecContext) {
+				res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(res).To(BeNil())
+			})
+
+			Context("when one init container is still running", func() {
+				BeforeEach(func() {
+					pod.Status.InitContainerStatuses = []corev1.ContainerStatus{
+						{
+							Name: "init-1",
+							State: corev1.ContainerState{
+								Terminated: &corev1.ContainerStateTerminated{
+									ExitCode: 0,
+									Reason:   "Completed",
+								},
+							},
+						},
+						{
+							Name: "init-2",
+							State: corev1.ContainerState{
+								Running: &corev1.ContainerStateRunning{},
+							},
+						},
+					}
+					mockCli = fake.NewClientBuilder().
+						WithScheme(k8scheme.BuildWithAllKnownScheme()).
+						WithObjects(cluster, pod).
+						Build()
+				})
+
+				It("should requeue and wait", func(ctx SpecContext) {
+					res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(res).ToNot(BeNil())
+					Expect(res.RequeueAfter).To(Equal(5 * time.Second))
+				})
+			})
+
+			Context("when one init container fails", func() {
+				BeforeEach(func() {
+					pod.Status.InitContainerStatuses = []corev1.ContainerStatus{
+						{
+							Name: "init-1",
+							State: corev1.ContainerState{
+								Terminated: &corev1.ContainerStateTerminated{
+									ExitCode: 0,
+									Reason:   "Completed",
+								},
+							},
+						},
+						{
+							Name: "init-2",
+							State: corev1.ContainerState{
+								Terminated: &corev1.ContainerStateTerminated{
+									ExitCode: 1,
+									Reason:   "Error",
+									Message:  "init failed",
+								},
+							},
+						},
+					}
+					mockCli = fake.NewClientBuilder().
+						WithScheme(k8scheme.BuildWithAllKnownScheme()).
+						WithObjects(cluster, pod).
+						Build()
+				})
+
+				It("should requeue with 10 second delay", func(ctx SpecContext) {
+					res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(res).ToNot(BeNil())
+					Expect(res.RequeueAfter).To(Equal(10 * time.Second))
+				})
+			})
+		})
+
+		Context("when sidecar init containers are present", func() {
+			BeforeEach(func() {
+				alwaysRestart := corev1.ContainerRestartPolicyAlways
+				pod.Spec.InitContainers = []corev1.Container{
+					{
+						Name: "restore-init",
+					},
+					{
+						Name:          "sidecar-init",
+						RestartPolicy: &alwaysRestart,
+					},
+				}
+				pod.Status.InitContainerStatuses = []corev1.ContainerStatus{
+					{
+						Name: "restore-init",
+						State: corev1.ContainerState{
+							Terminated: &corev1.ContainerStateTerminated{
+								ExitCode: 0,
+								Reason:   "Completed",
+							},
+						},
+					},
+					{
+						Name: "sidecar-init",
+						State: corev1.ContainerState{
+							Running: &corev1.ContainerStateRunning{},
+						},
+					},
+				}
+				mockCli = fake.NewClientBuilder().
+					WithScheme(k8scheme.BuildWithAllKnownScheme()).
+					WithObjects(cluster, pod).
+					Build()
+			})
+
+			It("should ignore sidecar containers and return success when non-sidecar containers complete",
+				func(ctx SpecContext) {
+					res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+					Expect(err).ToNot(HaveOccurred())
+					Expect(res).To(BeNil())
+				})
+		})
+
+		Context("when pod has owner references but init containers completed", func() {
+			BeforeEach(func() {
+				pod.OwnerReferences = []metav1.OwnerReference{
+					{
+						Name:       "some-controller",
+						Kind:       "Deployment",
+						UID:        "12345",
+						APIVersion: "apps/v1",
+					},
+				}
+				pod.Status.InitContainerStatuses = []corev1.ContainerStatus{
+					{
+						Name: "restore-init",
+						State: corev1.ContainerState{
+							Terminated: &corev1.ContainerStateTerminated{
+								ExitCode: 0,
+								Reason:   "Completed",
+							},
+						},
+					},
+				}
+				mockCli = fake.NewClientBuilder().
+					WithScheme(k8scheme.BuildWithAllKnownScheme()).
+					WithObjects(cluster, pod).
+					Build()
+			})
+
+			It("should still check and wait for init containers", func(ctx SpecContext) {
+				res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(res).To(BeNil())
+			})
+		})
+	})
+})
+
+var _ = Describe("isSidecarInitContainer", func() {
+	Context("when init container has no restart policy", func() {
+		It("should return false", func() {
+			container := &corev1.Container{
+				Name:          "test-init",
+				RestartPolicy: nil,
+			}
+			Expect(isSidecarInitContainer(container)).To(BeFalse())
+		})
+	})
+
+	Context("when init container has Always restart policy", func() {
+		It("should return true", func() {
+			alwaysRestart := corev1.ContainerRestartPolicyAlways
+			container := &corev1.Container{
+				Name:          "test-init",
+				RestartPolicy: &alwaysRestart,
+			}
+			Expect(isSidecarInitContainer(container)).To(BeTrue())
+		})
+	})
+
+	Context("when init container has OnFailure restart policy", func() {
+		It("should return false", func() {
+			onFailureRestart := corev1.ContainerRestartPolicyOnFailure
+			container := &corev1.Container{
+				Name:          "test-init",
+				RestartPolicy: &onFailureRestart,
+			}
+			Expect(isSidecarInitContainer(container)).To(BeFalse())
+		})
+	})
+
+	Context("when init container has Never restart policy", func() {
+		It("should return false", func() {
+			neverRestart := corev1.ContainerRestartPolicyNever
+			container := &corev1.Container{
+				Name:          "test-init",
+				RestartPolicy: &neverRestart,
+			}
+			Expect(isSidecarInitContainer(container)).To(BeFalse())
+		})
+	})
+})
+
+var _ = Describe("hasNonSidecarInitContainers", func() {
+	Context("when pod has no init containers", func() {
+		It("should return false", func() {
+			pod := &corev1.Pod{
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{},
+				},
+			}
+			Expect(hasNonSidecarInitContainers(pod)).To(BeFalse())
+		})
+	})
+
+	Context("when pod has only non-sidecar init containers", func() {
+		It("should return true", func() {
+			pod := &corev1.Pod{
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{
+							Name: "init-1",
+						},
+					},
+				},
+			}
+			Expect(hasNonSidecarInitContainers(pod)).To(BeTrue())
+		})
+	})
+
+	Context("when pod has only sidecar init containers", func() {
+		It("should return false", func() {
+			alwaysRestart := corev1.ContainerRestartPolicyAlways
+			pod := &corev1.Pod{
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{
+							Name:          "sidecar-init",
+							RestartPolicy: &alwaysRestart,
+						},
+					},
+				},
+			}
+			Expect(hasNonSidecarInitContainers(pod)).To(BeFalse())
+		})
+	})
+
+	Context("when pod has mixed init containers", func() {
+		It("should return true", func() {
+			alwaysRestart := corev1.ContainerRestartPolicyAlways
+			pod := &corev1.Pod{
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{
+							Name: "init-1",
+						},
+						{
+							Name:          "sidecar-init",
+							RestartPolicy: &alwaysRestart,
+						},
+					},
+				},
+			}
+			Expect(hasNonSidecarInitContainers(pod)).To(BeTrue())
+		})
+	})
+})
+
+var _ = Describe("getNonSidecarInitContainerStatuses", func() {
+	Context("when no init containers exist", func() {
+		It("should return empty slice", func() {
+			statuses := getNonSidecarInitContainerStatuses(
+				[]corev1.ContainerStatus{},
+				[]corev1.Container{},
+			)
+			Expect(statuses).To(BeEmpty())
+		})
+	})
+
+	Context("when only non-sidecar init containers exist", func() {
+		It("should return all statuses", func() {
+			initContainers := []corev1.Container{
+				{Name: "init-1"},
+				{Name: "init-2"},
+			}
+			statuses := []corev1.ContainerStatus{
+				{Name: "init-1", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}},
+				{Name: "init-2", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}},
+			}
+
+			result := getNonSidecarInitContainerStatuses(statuses, initContainers)
+			Expect(result).To(HaveLen(2))
+			Expect(result[0].Name).To(Equal("init-1"))
+			Expect(result[1].Name).To(Equal("init-2"))
+		})
+	})
+
+	Context("when only sidecar init containers exist", func() {
+		It("should return empty slice", func() {
+			alwaysRestart := corev1.ContainerRestartPolicyAlways
+			initContainers := []corev1.Container{
+				{
+					Name:          "sidecar-init",
+					RestartPolicy: &alwaysRestart,
+				},
+			}
+			statuses := []corev1.ContainerStatus{
+				{Name: "sidecar-init", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}},
+			}
+
+			result := getNonSidecarInitContainerStatuses(statuses, initContainers)
+			Expect(result).To(BeEmpty())
+		})
+	})
+
+	Context("when mixed init containers exist", func() {
+		It("should return only non-sidecar statuses", func() {
+			alwaysRestart := corev1.ContainerRestartPolicyAlways
+			initContainers := []corev1.Container{
+				{Name: "init-1"},
+				{Name: "sidecar-init", RestartPolicy: &alwaysRestart},
+				{Name: "init-2"},
+			}
+			statuses := []corev1.ContainerStatus{
+				{Name: "init-1", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}},
+				{Name: "sidecar-init", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}},
+				{Name: "init-2", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}},
+			}
+
+			result := getNonSidecarInitContainerStatuses(statuses, initContainers)
+			Expect(result).To(HaveLen(2))
+			Expect(result[0].Name).To(Equal("init-1"))
+			Expect(result[1].Name).To(Equal("init-2"))
+		})
+	})
+})
+
+var _ = Describe("ensureInitContainersAreCompleted - multiple pods", func() {
+	var (
+		mockCli k8client.Client
+		cluster *apiv1.Cluster
+		pod1    *corev1.Pod
+		pod2    *corev1.Pod
+	)
+
+	BeforeEach(func() {
+		cluster = &apiv1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "test-cluster",
+				Namespace: "default",
+			},
+		}
+	})
+
+	Context("when multiple pods exist with mixed init container states", func() {
+		BeforeEach(func() {
+			pod1 = &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod-1",
+					Namespace: "default",
+					Labels: map[string]string{
+						utils.ClusterLabelName: cluster.Name,
+						utils.PodRoleLabelName: string(utils.PodRoleInstance),
+					},
+				},
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{Name: "init-1"},
+					},
+				},
+				Status: corev1.PodStatus{
+					InitContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name: "init-1",
+							State: corev1.ContainerState{
+								Terminated: &corev1.ContainerStateTerminated{
+									ExitCode: 0,
+									Reason:   "Completed",
+								},
+							},
+						},
+					},
+				},
+			}
+
+			pod2 = &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod-2",
+					Namespace: "default",
+					Labels: map[string]string{
+						utils.ClusterLabelName: cluster.Name,
+						utils.PodRoleLabelName: string(utils.PodRoleInstance),
+					},
+				},
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{Name: "init-2"},
+					},
+				},
+				Status: corev1.PodStatus{
+					InitContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name: "init-2",
+							State: corev1.ContainerState{
+								Running: &corev1.ContainerStateRunning{},
+							},
+						},
+					},
+				},
+			}
+
+			mockCli = fake.NewClientBuilder().
+				WithScheme(k8scheme.BuildWithAllKnownScheme()).
+				WithObjects(cluster, pod1, pod2).
+				Build()
+		})
+
+		It("should requeue when at least one pod has running init containers", func(ctx SpecContext) {
+			res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).ToNot(BeNil())
+			Expect(res.RequeueAfter).To(Equal(5 * time.Second))
+		})
+	})
+
+	Context("when multiple pods exist and one has a failed init container", func() {
+		BeforeEach(func() {
+			pod1 = &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod-1",
+					Namespace: "default",
+					Labels: map[string]string{
+						utils.ClusterLabelName: cluster.Name,
+						utils.PodRoleLabelName: string(utils.PodRoleInstance),
+					},
+				},
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{Name: "init-1"},
+					},
+				},
+				Status: corev1.PodStatus{
+					InitContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name: "init-1",
+							State: corev1.ContainerState{
+								Terminated: &corev1.ContainerStateTerminated{
+									ExitCode: 0,
+									Reason:   "Completed",
+								},
+							},
+						},
+					},
+				},
+			}
+
+			pod2 = &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod-2",
+					Namespace: "default",
+					Labels: map[string]string{
+						utils.ClusterLabelName: cluster.Name,
+						utils.PodRoleLabelName: string(utils.PodRoleInstance),
+					},
+				},
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{Name: "init-2"},
+					},
+				},
+				Status: corev1.PodStatus{
+					InitContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name: "init-2",
+							State: corev1.ContainerState{
+								Terminated: &corev1.ContainerStateTerminated{
+									ExitCode: 1,
+									Reason:   "Error",
+									Message:  "restore failed",
+								},
+							},
+						},
+					},
+				},
+			}
+
+			mockCli = fake.NewClientBuilder().
+				WithScheme(k8scheme.BuildWithAllKnownScheme()).
+				WithObjects(cluster, pod1, pod2).
+				Build()
+		})
+
+		It("should requeue with 10 second delay when a pod has failed init container", func(ctx SpecContext) {
+			res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).ToNot(BeNil())
+			Expect(res.RequeueAfter).To(Equal(10 * time.Second))
+		})
+	})
+
+	Context("when multiple pods exist but only one matches the pod role label", func() {
+		BeforeEach(func() {
+			pod1 = &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod-1",
+					Namespace: "default",
+					Labels: map[string]string{
+						utils.ClusterLabelName: cluster.Name,
+						utils.PodRoleLabelName: string(utils.PodRoleInstance),
+					},
+				},
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{Name: "init-1"},
+					},
+				},
+				Status: corev1.PodStatus{
+					InitContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name: "init-1",
+							State: corev1.ContainerState{
+								Terminated: &corev1.ContainerStateTerminated{
+									ExitCode: 0,
+									Reason:   "Completed",
+								},
+							},
+						},
+					},
+				},
+			}
+
+			pod2 = &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod-2",
+					Namespace: "default",
+					Labels: map[string]string{
+						utils.ClusterLabelName: cluster.Name,
+						utils.PodRoleLabelName: "other-role",
+					},
+				},
+				Spec: corev1.PodSpec{
+					InitContainers: []corev1.Container{
+						{Name: "init-2"},
+					},
+				},
+				Status: corev1.PodStatus{
+					InitContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name: "init-2",
+							State: corev1.ContainerState{
+								Running: &corev1.ContainerStateRunning{},
+							},
+						},
+					},
+				},
+			}
+
+			mockCli = fake.NewClientBuilder().
+				WithScheme(k8scheme.BuildWithAllKnownScheme()).
+				WithObjects(cluster, pod1, pod2).
+				Build()
+		})
+
+		It("should ignore pods that don't match the pod role label", func(ctx SpecContext) {
+			res, err := ensureInitContainersAreCompleted(ctx, mockCli, cluster)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+		})
+	})
+})