Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: For a Pod with , kubelet would attempt to create a new sandbox for it even after all container succeeded. Logs as below: This issue is similar to which fixed the case of pods whose RestartPolicy are Never. It caused unnecessary CRI and CNI calls, confusing logs and conflicts between the routine that creates the new sandbox and the routine that kills the Pod. It's likely also the root cause of because sometimes the routine that creates the second sandbox and the routine that kills unwanted pod run concurrently, then the netns would be gone when calling CNI. What you expected to happen: kubelet shouldn't create a new sandbox for pod with RestartPolicyOnFailure if all containers succeeded How to reproduce it (as minimally and precisely as possible): Create a pod like this: Anything else we need to know?: Environment: Kubernetes version (use ): v1.18.2 Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig node
Created which may fix this.
/assign

Discussion:
changes.CreateSandbox = false return changes } <ins> // Get the containers to start, excluding the ones that succeeded if RestartPolicy is OnFailure. var containersToStart []int for idx, c := range pod.Spec.Containers { if pod.Spec.RestartPolicy == v1.RestartPolicyOnFailure && containerSucceeded(&c, podStatus) { continue } containersToStart = append(containersToStart, idx) } // We should not create a sandbox for a Pod if initialization is done and there is no container to start. if len(containersToStart) == 0 { _, _, done := findNextInitContainerToRun(pod, podStatus) if done { changes.CreateSandbox = false return changes } } </ins> if len(pod.Spec.InitContainers) != 0 { // Pod has init containers, return the first one. changes.NextInitContainerToStart = &pod.Spec.InitContainers[0] return changes } <del> // Start all containers by default but exclude the ones that succeeded if // RestartPolicy is OnFailure. for idx, c := range pod.Spec.Containers { if containerSucceeded(&c, podStatus) && pod.Spec.RestartPolicy == v1.RestartPolicyOnFailure { continue } changes.ContainersToStart = append(changes.ContainersToStart, idx) } </del> <ins> changes.ContainersToStart = containersToStart </ins> return changes }

ContainersToKill:  map[kubecontainer.ContainerID]containerToKillInfo{}, }, }, <ins> "Verify we do not create a pod sandbox if no ready sandbox for pod with RestartPolicy=OnFailure and all containers succeeded": { mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, mutateStatusFn: func(status *kubecontainer.PodStatus) { // no ready sandbox status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY status.SandboxStatuses[0].Metadata.Attempt = uint32(1) // all containers succeeded for i := range status.ContainerStatuses { status.ContainerStatuses[i].State = kubecontainer.ContainerStateExited status.ContainerStatuses[i].ExitCode = 0 } }, actions: podActions{ SandboxID:         baseStatus.SandboxStatuses[0].Id, Attempt:           uint32(2), CreateSandbox:     false, KillPod:           true, ContainersToStart: []int{}, ContainersToKill:  map[kubecontainer.ContainerID]containerToKillInfo{}, }, }, </ins> "Verify we create a pod sandbox if no ready sandbox for pod with RestartPolicy=Never and no containers have ever been created": { mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever

ContainersToKill:         getKillMapWithInitContainers(basePod, baseStatus, []int{}), }, }, <ins> "Pod sandbox not ready, init container failed, and RestartPolicy == OnFailure; create a new pod sandbox": { mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, mutateStatusFn: func(status *kubecontainer.PodStatus) { status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY status.ContainerStatuses[2].ExitCode = 137 }, actions: podActions{ KillPod:                  true, CreateSandbox:            true, SandboxID:                baseStatus.SandboxStatuses[0].Id, Attempt:                  uint32(1), NextInitContainerToStart: &basePod.Spec.InitContainers[0], ContainersToStart:        []int{}, ContainersToKill:         getKillMapWithInitContainers(basePod, baseStatus, []int{}), }, }, </ins> } { pod, status := makeBasePodAndStatusWithInitContainers() if test.mutatePodFn != nil {

EphemeralContainersToStart: []int{0}, }, }, <ins> "Create a new pod sandbox if the pod sandbox is dead, init container failed and RestartPolicy == OnFailure": { mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure }, mutateStatusFn: func(status *kubecontainer.PodStatus) { status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY status.ContainerStatuses = status.ContainerStatuses[3:] status.ContainerStatuses[0].ExitCode = 137 }, actions: podActions{ KillPod:                  true, CreateSandbox:            true, SandboxID:                baseStatus.SandboxStatuses[0].Id, Attempt:                  uint32(1), NextInitContainerToStart: &basePod.Spec.InitContainers[0], ContainersToStart:        []int{}, ContainersToKill:         getKillMapWithInitContainers(basePod, baseStatus, []int{}), }, }, </ins> "Kill pod and do not restart ephemeral container if the pod sandbox is dead": { mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyAlways }, mutateStatusFn: func(status *kubecontainer.PodStatus) {

visibility = ["//visibility:public"], deps = [ "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", <ins> "//pkg/kubelet/events:go_default_library", </ins> "//pkg/kubelet/runtimeclass/testing:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/util/slice:go_default_library",

v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> "k8s.io/apimachinery/pkg/fields" </ins> "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" <ins> "k8s.io/kubernetes/pkg/kubelet/events" </ins> "k8s.io/kubernetes/test/e2e/framework" e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet" e2epod "k8s.io/kubernetes/test/e2e/framework/pod"

framework.Failf("%d errors:n%v", len(errs), strings.Join(messages, "n")) } }) <ins> }) framework.KubeDescribe("Pod Container lifecycle", func() { var podClient *framework.PodClient ginkgo.BeforeEach(func() { podClient = f.PodClient() }) ginkgo.It("should not create extra sandbox if all containers are done", func() { ginkgo.By("creating the pod that should always exit 0") name := "pod-always-succeed" + string(uuid.NewUUID()) image := imageutils.GetE2EImage(imageutils.BusyBox) pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: v1.PodSpec{ RestartPolicy: v1.RestartPolicyOnFailure, InitContainers: []v1.Container{ { Name:  "foo", Image: image, Command: []string{ "/bin/true", }, }, }, Containers: []v1.Container{ { Name:  "bar", Image: image, Command: []string{ "/bin/true", }, }, }, }, } ginkgo.By("submitting the pod to kubernetes") createdPod := podClient.Create(pod) defer func() { ginkgo.By("deleting the pod") podClient.Delete(context.TODO(), pod.Name, metav1.DeleteOptions{}) }() framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)) var eventList *v1.EventList var err error ginkgo.By("Getting events about the pod") framework.ExpectNoError(wait.Poll(time.Second*2, time.Second*60, func() (bool, error) { selector := fields.Set{ "involvedObject.kind":      "Pod", "involvedObject.uid":       string(createdPod.UID), "involvedObject.namespace": f.Namespace.Name, "source":                   "kubelet", }.AsSelector().String() options := metav1.ListOptions{FieldSelector: selector} eventList, err = f.ClientSet.CoreV1().Events(f.Namespace.Name).List(context.TODO(), options) if err != nil { return false, err } if len(eventList.Items) > 0 { return true, nil } return false, nil })) ginkgo.By("Checking events about the pod") for _, event := range eventList.Items { if event.Reason == events.SandboxChanged { framework.Fail("Unexpected SandboxChanged event") } } }) </ins> }) })