Change:
In a situation where during the init phase of a , a native sidecar never becomes ready (because its is failing), then the is terminated, if there are multiple startup containers, a termination signal is never received by the init container and the Pod is stuck (until is hit). All running initContainer to receive the termination signal correctly. Create a with this bash script as an entrypoint: Create a deployment with two init containers (one normal one sidecar). Add an probe which means the startupProbe never passes Create the deployment, and it'll be stuck in Init, as you'd expect: the pod and you'll see the exit signal has been sent to : Delete the deployment / clean everything up. Now, add a third : As before, the process will be stuck on because of the bad probe: However when you delete the pod this time, no exit signal is sent to , the pod is now stuck until is reached. No response GKE v1.29.4-gke. Core OS on GCP <details</details<details</details<details</details
/sig node
/assign
/triage accepted confirmed that the scenario is reproduced. The problem is caused by the fact that the native sidecars should terminate in reverse order. I'll fix the code to ignore not-running init containers. /cc
Awesome, thank you :)
sig-node meeting notes: /triage accepted /priority important-soon
FYI: confirmed that the SITERM is sent to the containers after terminationGracePeriodSeconds (and wait for the application for 2 seconds before sending SIGKILL) will fix it anyway with

Discussion:
to.terminated[c.Name] = channel mainContainerChannels = append(mainContainerChannels, channel) <del> // if its not a running container, pre-close the channel so nothing waits on it </del> <ins> // if it's not a running container, pre-close the channel so nothing // waits on it </ins> if _, isRunning := runningContainers[c.Name]; !isRunning { close(channel) }

// get the init containers in reverse order ic := pod.Spec.InitContainers[len(pod.Spec.InitContainers)-i-1] <del> to.terminated[ic.Name] = make(chan struct{}) </del> <ins> channel := make(chan struct{}) to.terminated[ic.Name] = channel // if it's not a running container, pre-close the channel so nothing // waits on it if _, isRunning := runningContainers[ic.Name]; !isRunning { close(channel) } </ins> if types.IsRestartableInitContainer(&ic) { // sidecars need to wait for all main containers to exit

// should delete quickly and not try to start/wait on any sidecars since they never started gomega.Expect(deleteTime).To(gomega.BeNumerically("<", grace+buffer), fmt.Sprintf("should delete in < %d seconds, took %f", grace+buffer, deleteTime)) }) <ins> f.It("should terminate restartable init containers gracefully if there is a non-started restartable init container", func(ctx context.Context) { init1 := "init-1" restartableInit2 := "restartable-init-2" restartableInit3 := "restartable-init-3" regular1 := "regular-1" podTerminationGracePeriodSeconds := int64(180) containerTerminationSeconds := 1 pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "terminate-restartable-init-gracefully", }, Spec: v1.PodSpec{ TerminationGracePeriodSeconds: &podTerminationGracePeriodSeconds, RestartPolicy:                 v1.RestartPolicyNever, InitContainers: []v1.Container{ { Name:  init1, Image: busyboxImage, Command: ExecCommand(init1, execCommand{ Delay:              1, TerminationSeconds: 5, ExitCode:           0, }), }, { Name:  restartableInit2, Image: busyboxImage, Command: ExecCommand(restartableInit2, execCommand{ Delay:              600, TerminationSeconds: containerTerminationSeconds, ExitCode:           0, }), StartupProbe: &v1.Probe{ FailureThreshold: 600, ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ Command: []string{"false"}, }, }, }, RestartPolicy: &containerRestartPolicyAlways, }, { Name:  restartableInit3, Image: busyboxImage, Command: ExecCommand(restartableInit3, execCommand{ Delay:              600, TerminationSeconds: 1, ExitCode:           0, }), RestartPolicy: &containerRestartPolicyAlways, }, }, Containers: []v1.Container{ { Name:  regular1, Image: busyboxImage, Command: ExecCommand(regular1, execCommand{ Delay:              600, TerminationSeconds: 1, ExitCode:           0, }), }, }, }, } preparePod(pod) client := e2epod.NewPodClient(f) pod = client.Create(ctx, pod) err := e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "the second init container is running but not started", 2*time.Minute, func(pod *v1.Pod) (bool, error) { if pod.Status.Phase != v1.PodPending { return false, fmt.Errorf("pod should be in pending phase") } if len(pod.Status.InitContainerStatuses) != 3 { return false, fmt.Errorf("pod should have the same number of statuses as init containers") } containerStatus := pod.Status.InitContainerStatuses[1] return containerStatus.State.Running != nil && (containerStatus.Started == nil || *containerStatus.Started == false), nil }) framework.ExpectNoError(err) ginkgo.By("Deleting the pod") err = client.Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &podTerminationGracePeriodSeconds}) framework.ExpectNoError(err) ginkgo.By("Waiting for the pod to terminate gracefully before its terminationGracePeriodSeconds") err = e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, // The duration should be less than the pod's // terminationGracePeriodSeconds while adding a buffer(60s) to the // container termination seconds(1s) to account for the time it // takes to delete the pod. time.Duration(containerTerminationSeconds+60)*time.Second) framework.ExpectNoError(err, "the pod should be deleted before its terminationGracePeriodSeconds if the restartalbe init containers get termination signal correctly") }) </ins> }) var _ = SIGDescribe(nodefeature.SidecarContainers, framework.WithSerial(), "Containers Lifecycle", func() {