Repository: kubernetes/kubernetes (language: go)

Issue:
Termination signals not sent to Native Sidecars when multiple Native Sidecars
### What happened?

In a situation where during the init phase of a `Pod`, a native sidecar never becomes ready (because its `startupProbe` is failing), then the `Pod` is terminated, if there are multiple startup containers, a termination signal is never received by the init container and the Pod is stuck (until `terminationGracePeriodSeconds` is hit).

### What did you expect to happen?

All running initContainer to receive the termination signal correctly.  

### How can we reproduce it (as minimally and precisely as possible)?

Create a `Dockerfile` with this bash script as an entrypoint:

```
#!/bin/bash
set -e 
trap 'echo "saw an exit signal"; sleep 5; exit' EXIT

while true; do
  echo "hello"
  sleep 1
done
```

Create a deployment with two init containers (one normal one sidecar).  Add an probe which means the startupProbe never passes

```
apiVersion: apps/v1
kind: Deployment
metadata:
  name: karl-test-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app2: karl-test
  template:
    metadata:
      labels:
        app2: karl-test 
    spec:
      securityContext:
        seccompProfile:
          type: RuntimeDefault
      terminationGracePeriodSeconds: 600
      initContainers:
      - name: init-1
        image: eu.gcr.io/at-artefacts/karl-test
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c", "echo 'init container 1'"]
      - name: init-2
        image: eu.gcr.io/at-artefacts/karl-test
        imagePullPolicy: IfNotPresent
        restartPolicy: Always 
        startupProbe:                                                                                                                                                                                                          
          failureThreshold: 600 
          httpGet:
            path: /whatever
            port: 1234
            scheme: HTTP
          periodSeconds: 1
          successThreshold: 1
          timeoutSeconds: 3
      containers:
      - name: karl-test
        image: eu.gcr.io/at-artefacts/karl-test
        imagePullPolicy: IfNotPresent
```

Create the deployment, and it'll be stuck in Init, as you'd expect:

```
karl-test-deployment-c979948d8-xghf4   0/2     Init:1/2   0          5s
```

`kubectl delete` the pod and you'll see the exit signal has been sent to `init-2`:
```
hello
hello
hello
saw an exit signal
```

Delete the deployment / clean everything up.  Now, add a third `initContainer`:

```
apiVersion: apps/v1
kind: Deployment
metadata:
  name: karl-test-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app2: karl-test
  template:
    metadata:
      labels:
        app2: karl-test 
    spec:
      securityContext:
        seccompProfile:
          type: RuntimeDefault
      terminationGracePeriodSeconds: 600
      initContainers:
      - name: init-1
        image: eu.gcr.io/at-artefacts/karl-test
        imagePullPolicy: IfNotPresent
        command: ["sh", "-c", "echo 'init container 1'"]
      - name: init-2
        image: eu.gcr.io/at-artefacts/karl-test
        imagePullPolicy: IfNotPresent
        restartPolicy: Always 
        startupProbe:                                                                                                                                                                                                          
          failureThreshold: 600 
          httpGet:
            path: /whatever
            port: 1234
            scheme: HTTP
          periodSeconds: 1
          successThreshold: 1
          timeoutSeconds: 3
      - name: init-3
        image: eu.gcr.io/at-artefacts/karl-test
        imagePullPolicy: IfNotPresent
        restartPolicy: Always 
        startupProbe:                                                                                                                                                                                                          
          failureThreshold: 600 
          httpGet:
            path: /whatever
            port: 1234
            scheme: HTTP
          periodSeconds: 1
          successThreshold: 1
          timeoutSeconds: 3
      containers:
      - name: karl-test
        image: eu.gcr.io/at-artefacts/karl-test
        imagePullPolicy: IfNotPresent
```

As before, the `init` process will be stuck on `init-2` because of the bad probe:
```
karl-test-deployment-5c547f8f86-s8nqh   0/3     Init:1/3   0          1s
```

However when you delete the pod this time, no exit signal is sent to `init-2`, the pod is now stuck until `terminationGracePeriodSeconds` is reached.

### Anything else we need to know?

_No response_

### Kubernetes version

```
Client Version: v1.28.5
Server Version: v1.29.4-gke.1043002
```

### Cloud provider

GKE v1.29.4-gke.1043002

### OS version

Core OS on GCP

### Install tools

<details>

</details>


### Container runtime (CRI) and version (if applicable)

<details>

</details>


### Related plugins (CNI, CSI, ...) and versions (if applicable)

<details>

</details>



Discussion / hints:
/sig node

/assign

/triage accepted

confirmed that the scenario is reproduced.

The problem is caused by the fact that the native sidecars should terminate in reverse order.
I'll fix the code to ignore not-running init containers.

/cc @SergeyKanzhelev @matthyx 
Awesome, thank you :)

Fix patch:
diff --git a/pkg/kubelet/kuberuntime/kuberuntime_termination_order.go b/pkg/kubelet/kuberuntime/kuberuntime_termination_order.go
index ec417ecd76520..0e8a2a6adf8a2 100644
--- a/pkg/kubelet/kuberuntime/kuberuntime_termination_order.go
+++ b/pkg/kubelet/kuberuntime/kuberuntime_termination_order.go
@@ -56,7 +56,8 @@ func newTerminationOrdering(pod *v1.Pod, runningContainerNames []string) *termin
 		to.terminated[c.Name] = channel
 		mainContainerChannels = append(mainContainerChannels, channel)
 
-		// if its not a running container, pre-close the channel so nothing waits on it
+		// if it's not a running container, pre-close the channel so nothing
+		// waits on it
 		if _, isRunning := runningContainers[c.Name]; !isRunning {
 			close(channel)
 		}
@@ -67,7 +68,14 @@ func newTerminationOrdering(pod *v1.Pod, runningContainerNames []string) *termin
 		// get the init containers in reverse order
 		ic := pod.Spec.InitContainers[len(pod.Spec.InitContainers)-i-1]
 
-		to.terminated[ic.Name] = make(chan struct{})
+		channel := make(chan struct{})
+		to.terminated[ic.Name] = channel
+
+		// if it's not a running container, pre-close the channel so nothing
+		// waits on it
+		if _, isRunning := runningContainers[ic.Name]; !isRunning {
+			close(channel)
+		}
 
 		if types.IsRestartableInitContainer(&ic) {
 			// sidecars need to wait for all main containers to exit


Test patch:
diff --git a/test/e2e_node/container_lifecycle_test.go b/test/e2e_node/container_lifecycle_test.go
index 6caf25bfca158..61e2a5db96841 100644
--- a/test/e2e_node/container_lifecycle_test.go
+++ b/test/e2e_node/container_lifecycle_test.go
@@ -3115,6 +3115,107 @@ var _ = SIGDescribe(nodefeature.SidecarContainers, "Containers Lifecycle", func(
 		// should delete quickly and not try to start/wait on any sidecars since they never started
 		gomega.Expect(deleteTime).To(gomega.BeNumerically("<", grace+buffer), fmt.Sprintf("should delete in < %d seconds, took %f", grace+buffer, deleteTime))
 	})
+
+	f.It("should terminate restartable init containers gracefully if there is a non-started restartable init container", func(ctx context.Context) {
+		init1 := "init-1"
+		restartableInit2 := "restartable-init-2"
+		restartableInit3 := "restartable-init-3"
+		regular1 := "regular-1"
+
+		podTerminationGracePeriodSeconds := int64(180)
+		containerTerminationSeconds := 1
+
+		pod := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "terminate-restartable-init-gracefully",
+			},
+			Spec: v1.PodSpec{
+				TerminationGracePeriodSeconds: &podTerminationGracePeriodSeconds,
+				RestartPolicy:                 v1.RestartPolicyNever,
+				InitContainers: []v1.Container{
+					{
+						Name:  init1,
+						Image: busyboxImage,
+						Command: ExecCommand(init1, execCommand{
+							Delay:              1,
+							TerminationSeconds: 5,
+							ExitCode:           0,
+						}),
+					},
+					{
+						Name:  restartableInit2,
+						Image: busyboxImage,
+						Command: ExecCommand(restartableInit2, execCommand{
+							Delay:              600,
+							TerminationSeconds: containerTerminationSeconds,
+							ExitCode:           0,
+						}),
+						StartupProbe: &v1.Probe{
+							FailureThreshold: 600,
+							ProbeHandler: v1.ProbeHandler{
+								Exec: &v1.ExecAction{
+									Command: []string{"false"},
+								},
+							},
+						},
+						RestartPolicy: &containerRestartPolicyAlways,
+					},
+					{
+						Name:  restartableInit3,
+						Image: busyboxImage,
+						Command: ExecCommand(restartableInit3, execCommand{
+							Delay:              600,
+							TerminationSeconds: 1,
+							ExitCode:           0,
+						}),
+						RestartPolicy: &containerRestartPolicyAlways,
+					},
+				},
+				Containers: []v1.Container{
+					{
+						Name:  regular1,
+						Image: busyboxImage,
+						Command: ExecCommand(regular1, execCommand{
+							Delay:              600,
+							TerminationSeconds: 1,
+							ExitCode:           0,
+						}),
+					},
+				},
+			},
+		}
+
+		preparePod(pod)
+
+		client := e2epod.NewPodClient(f)
+		pod = client.Create(ctx, pod)
+
+		err := e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "the second init container is running but not started", 2*time.Minute, func(pod *v1.Pod) (bool, error) {
+			if pod.Status.Phase != v1.PodPending {
+				return false, fmt.Errorf("pod should be in pending phase")
+			}
+			if len(pod.Status.InitContainerStatuses) != 3 {
+				return false, fmt.Errorf("pod should have the same number of statuses as init containers")
+			}
+			containerStatus := pod.Status.InitContainerStatuses[1]
+			return containerStatus.State.Running != nil &&
+				(containerStatus.Started == nil || *containerStatus.Started == false), nil
+		})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("Deleting the pod")
+		err = client.Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &podTerminationGracePeriodSeconds})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("Waiting for the pod to terminate gracefully before its terminationGracePeriodSeconds")
+		err = e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace,
+			// The duration should be less than the pod's
+			// terminationGracePeriodSeconds while adding a buffer(60s) to the
+			// container termination seconds(1s) to account for the time it
+			// takes to delete the pod.
+			time.Duration(containerTerminationSeconds+60)*time.Second)
+		framework.ExpectNoError(err, "the pod should be deleted before its terminationGracePeriodSeconds if the restartalbe init containers get termination signal correctly")
+	})
 })
 
 var _ = SIGDescribe(nodefeature.SidecarContainers, framework.WithSerial(), "Containers Lifecycle", func() {