Change:
} } }) <ins> // Regression test for https://issues.k8s.io/116925 ginkgo.It("should delete pods which are marked as terminal and have a deletion timestamp set after restart", func(ctx context.Context) { podName := "terminal-restart-pod" + string(uuid.NewUUID()) gracePeriod := int64(30) podSpec := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, }, Spec: v1.PodSpec{ TerminationGracePeriodSeconds: &gracePeriod, RestartPolicy:                 v1.RestartPolicyNever, Containers: []v1.Container{ { Name:    podName, Image:   imageutils.GetE2EImage(imageutils.BusyBox), Command: []string{"sh", "-c"}, Args: []string{` sleep 9999999 & PID=$! _term () { kill $PID echo "Caught SIGTERM!" } trap _term SIGTERM wait $PID trap - TERM # Wait for the long running sleep to exit wait $PID exit 0 `, }, }, }, }, }) ginkgo.By(fmt.Sprintf("Creating a pod (%v/%v) with restart policy: %v", f.Namespace.Name, podName, podSpec.Spec.RestartPolicy)) pod := e2epod.NewPodClient(f).Create(ctx, podSpec) ginkgo.By(fmt.Sprintf("Waiting for the pod (%v/%v) to be running", f.Namespace.Name, pod.Name)) err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name) framework.ExpectNoError(err, "Failed to await for the pod to be running: (%v/%v)", f.Namespace.Name, pod.Name) w := &cache.ListWatch{ WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { return f.ClientSet.CoreV1().Pods(f.Namespace.Name).Watch(ctx, options) }, } podsList, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(ctx, metav1.ListOptions{}) framework.ExpectNoError(err, "Failed to list pods in namespace: %s", f.Namespace.Name) ginkgo.By(fmt.Sprintf("Deleting the pod (%v/%v) to set a deletion timestamp", pod.Namespace, pod.Name)) time.Sleep(time.Second) err = e2epod.NewPodClient(f).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}) framework.ExpectNoError(err, "Failed to delete the pod: %q", pod.Name) ctxUntil, cancel := context.WithTimeout(ctx, f.Timeouts.PodStart) defer cancel() ginkgo.By(fmt.Sprintf("Started watch for pod (%v/%v) to enter succeeded phase", pod.Namespace, pod.Name)) _, err = watchtools.Until(ctxUntil, podsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if pod, ok := event.Object.(*v1.Pod); ok { found := pod.ObjectMeta.Name == podName && pod.ObjectMeta.Namespace == f.Namespace.Name && pod.Status.Phase == v1.PodSucceeded if !found { ginkgo.By(fmt.Sprintf("Observed Pod (%s/%s) in phase %v", pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, pod.Status.Phase)) return false, nil } ginkgo.By(fmt.Sprintf("Found Pod (%s/%s) in phase %v", pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, pod.Status.Phase)) return found, nil } ginkgo.By(fmt.Sprintf("Observed event: %+v", event.Object)) return false, nil }) ginkgo.By("Ended watch for pod entering succeeded phase") framework.ExpectNoError(err, "failed to see event that pod (%s/%s) enter succeeded phase: %v", pod.Namespace, pod.Name, err) // As soon as the pod enters succeeded phase (detected by the watch above); kill the kubelet. // This is a bit racy, but the goal is to stop the kubelet before the kubelet is able to delete the pod from the API-sever in order to repro https://issues.k8s.io/116925 ginkgo.By("Stopping the kubelet") startKubelet := stopKubelet() // wait until the kubelet health check will fail gomega.Eventually(ctx, func() bool { return kubeletHealthCheck(kubeletHealthCheckURL) }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeFalse()) ginkgo.By("Starting the kubelet") startKubelet() // wait until the kubelet health check will succeed gomega.Eventually(ctx, func() bool { return kubeletHealthCheck(kubeletHealthCheckURL) }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrue()) // Wait for the Kubelet to be ready. gomega.Eventually(ctx, func(ctx context.Context) bool { nodes, err := e2enode.TotalReady(ctx, f.ClientSet) framework.ExpectNoError(err) return nodes == 1 }, time.Minute, f.Timeouts.Poll).Should(gomega.BeTrue()) ginkgo.By(fmt.Sprintf("After the kubelet is restarted, verify the pod (%s/%s) is deleted by kubelet", pod.Namespace, pod.Name)) gomega.Eventually(ctx, func(ctx context.Context) error { return checkMirrorPodDisappear(ctx, f.ClientSet, pod.Name, pod.Namespace) }, f.Timeouts.PodDelete, f.Timeouts.Poll).Should(gomega.BeNil()) }) </ins> }) })

Discussion:
New TestGrid alerts at since commit/. Found 2 failing rows in node-kubelet-serial-containerd: 2 new, 0 other 2 new Test Failures in node-kubelet-serial-containerd. Test has failed 3 times since it last passed at commit/. It started failing between commit/ (Fri Mar 24 12:02 PDT) and commit/ (Fri Mar 24 16:03 PDT): . Test has failed 3 times since it last passed at commit/. It started failing between commit/ (Fri Mar 24 12:02 PDT) and commit/ (Fri Mar 24 16:03 PDT): . Not sure suspect No response /sig node release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/kind flake /remove-kind failing-test See
/remove-sig release
Taking a look at The test failed due to: so it means that the pod was not terminated and deleted properly in the test that ran prior. The is the one creating this pod, but I would expect that as part of the test teardown, the namespace is deleted (and thus the pod should be deleted).
The services log show that the kubelet never sent the request to delete the pod, that is strange...
From the build-logs, the test execution for when the pod was created: Kubelet logs for the pod - Full logs are here -
Here's the termination part of the logs:
It looks like to me the issue here is that the kubelet is being restarted in a bit of an unfortunate time resulting in at least two related issues: The pod status has been updated on the api-server to terminal just prior to the kubelet restarting. The status manager did not yet complete . Then after kubelet restarts and comes back up, it attempts to add this pod , (even though it is in terminal phase). This seems like the first issue: if the pod phase is terminal on the API server, we probably should not add it back.

The pod worker receives the , but the pod worker quickly terminates (). I am assuming this is because of , i.e. based on pod cache lookup the pod is terminal. The side affect of this is that the status manager never receives the signal that the pod is (which is normally done as part of . This is because in this case, we never call which sets to be true. As a result, I think will never return true, so this pod will never be able to be deleted.
I have made a test which will reproduce this behavior in a node e2e, see It can be reproduced by starting a pod and then deleting it and triggering kubelet restart at the right time. A watch is setup to trigger a kubelet restart as soon as the pod enters the succeeded phase. If the kubelet restart happens before the kubelet is able to delete the pod on the API server, but after the phase is updated, then we trigger the issue. The pod will be re- to kubelet after restart, but will never be deleted on the API server despite it having a deletion timestamp set and being in terminal phase. Here are logs from the and running the above repro node e2e test. Here is the kubelet logs portion prior to the restart: and after restart: It also looks like we are hitting another invalid state in based on the log assertion: I tried the same test on 1.26 and it passes. This is because on 1.26, the status manager phase to terminal happens at roughly at the same time when we delete the pod. On 1.27+, after , the status phase to terminal can happen earlier and the status manager can skip deleting the pod because . In this case, after start, will never be true, so the pod never will get deleted.
But in general pods can be in terminal phase and not be deleted (like plain pods and pods from jobs). They would eventually be deleted by the pod GC otherwise. Is this a bug in the device plugin?
That is true, but the issue here is that the pods have a deletion timestamp set as well. If the deletion timestamp is set and the kubelet is up, it is the kubelet's responsibility to terminate and delete those pods.

This doesn't have anything with device plugin, rather we caught the issue due to the device plugin test which asserts that all pods are terminated and deleted prior to starting the test.
I have a WIP fix for this issue in
That makes sense
Should we aim 1.27 with this fix? This seems like an important regression.
I discussed this with yesterday and we agreed that this can wait until 1.27.1 Clients should be able to handle terminated pods. Worst case scenario, the pod GC would take care of these pods eventually.
Cherrypicked to v1.27 (to be released in 1.27.2) in