Change:
The scheduler cache entered a corrupted state where a Pod, according to the cache dump, was nominated to node A, while the status said it was nominated to node B. This lead to unschedulable low priority pods due to the phantom pod in the nominator cache. The cache to match the Pod status Setup: two nodes with all the resources used. A high priority pod comes in that is able to preempt at least one pod. This might cause these sequence of events X doesn't fit, it gets nominated to and back to scheduling queue. start a second scheduling attempt for Pod X. get a Pod update for the . This puts the pod back into the scheduling queue. second scheduling attempt finishes and schedules the pod successfully in start a third scheduling attempt for Pod X (due to 3) get a Pod update for the . This should have cleared the nominated node in the cache, but there is nothing currently there. the third scheduling attempt, the pod doesn't fit, so the scheduler nominates it to . Nothing clears this nomination afterwards Usually step 2 happens after 3, so everything works as expected. But in a busy cluster, 3 might be delayed. No response <detailsGKE </details<details</details<details</details<details</details<details</details
/sig scheduling cc
This Pod actually got scheduled: The cache dump was obtained much later at : I think the problem is that we got an update for the Pod (probably the nominated node itself), scheduled it again and assigned a second nominated node for it. The second scheduling attempt failed, but we didn't clear the cache.
/triage accepted
/priority important-soon
Sounds like is related?
It could be the same, or it could be another bug. Note that this is a second scheduling attempt for a Pod that was already scheduled.
My theory of events: X doesn't fit, it gets nominated to and back to scheduling queue. start a second scheduling attempt for Pod X. get a Pod update for the . This puts the pod back into the scheduling queue. second scheduling attempt finishes and schedules the pod successfully in start a third scheduling attempt for Pod X (due to 3) get a Pod update for the . This should have cleared the nominated node in the cache, but there is nothing currently there. the third scheduling attempt, the pod doesn't fit, so the scheduler nominates it to . Not sure why this pod update fails, but nevertheless, we have a corrupted cache at this point.
thanks for raising up this and the fix. Could you update the "How can we reproduce it (as minimally and precisely as possible)?" section a bit? Per our offline conversation, the issue will leave a "phantom" pod holding a number of resources in the internal nominator cache. And due to that unnecessary resource reservation, low-priority pods may not get scheduled while they should have been. This might be the key trigger of this issue. In practice, it can be caused by a (way-too-quick) movePodBackToActiveQ call (due to relevant/irrelevant events, or backoff's completion) after step 1 and before step 3.
Updated the description. I will update the PRs on Monday.

Discussion:
} if npm.podLister != nil { <del> // If the pod is not alive, don't contain it. if _, err := npm.podLister.Pods(pi.Pod.Namespace).Get(pi.Pod.Name); err != nil { klog.V(4).InfoS("Pod doesn't exist in podLister, aborting adding it to the nominator", "pod", klog.KObj(pi.Pod)) </del> <ins> // If the pod was removed or if it was already scheduled, don't nominate it. updatedPod, err := npm.podLister.Pods(pi.Pod.Namespace).Get(pi.Pod.Name) if err != nil { klog.V(4).InfoS("Pod doesn't exist in podLister, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod)) return } if updatedPod.Spec.NodeName != "" { klog.V(4).InfoS("Pod is already scheduled to a node, aborted adding it to the nominator", "pod", klog.KObj(pi.Pod), "node", updatedPod.Spec.NodeName) </ins> return } }

TestEvent    = framework.ClusterEvent{Resource: "test"} NodeAllEvent = framework.ClusterEvent{Resource: framework.Node, ActionType: framework.All} EmptyEvent   = framework.ClusterEvent{} <del> ) var lowPriority, midPriority, highPriority = int32(0), int32(100), int32(1000) var mediumPriority = (lowPriority + highPriority) / 2 var highPriorityPodInfo, highPriNominatedPodInfo, medPriorityPodInfo, unschedulablePodInfo = framework.NewPodInfo(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "hpp", Namespace: "ns1", UID:       "hppns1", }, Spec: v1.PodSpec{ Priority: &highPriority, }, }), framework.NewPodInfo(&v1.Pod{ </del> <ins> lowPriority, midPriority, highPriority = int32(0), int32(100), int32(1000) mediumPriority                         = (lowPriority + highPriority) / 2 highPriorityPodInfo = framework.NewPodInfo(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "hpp", Namespace: "ns1", UID:       "hppns1", }, Spec: v1.PodSpec{ Priority: &highPriority, }, }) highPriNominatedPodInfo = framework.NewPodInfo(&v1.Pod{ </ins> ObjectMeta: metav1.ObjectMeta{ Name:      "hpp", Namespace: "ns1",

Status: v1.PodStatus{ NominatedNodeName: "node1", }, <del> }), framework.NewPodInfo(&v1.Pod{ </del> <ins> }) medPriorityPodInfo = framework.NewPodInfo(&v1.Pod{ </ins> ObjectMeta: metav1.ObjectMeta{ Name:      "mpp", Namespace: "ns2",

Status: v1.PodStatus{ NominatedNodeName: "node1", }, <del> }), framework.NewPodInfo(&v1.Pod{ </del> <ins> }) unschedulablePodInfo = framework.NewPodInfo(&v1.Pod{ </ins> ObjectMeta: metav1.ObjectMeta{ Name:      "up", Namespace: "ns1",

NominatedNodeName: "node1", }, }) <ins> nonExistentPodInfo = framework.NewPodInfo(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "ne", Namespace: "ns1", UID:       "nens1", }, }) scheduledPodInfo = framework.NewPodInfo(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "sp", Namespace: "ns1", UID:       "spns1", }, Spec: v1.PodSpec{ NodeName: "foo", }, }) ) </ins> func getUnschedulablePod(p *PriorityQueue, pod *v1.Pod) *v1.Pod { pInfo := p.unschedulablePods.get(pod)

} func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { <del> objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod} </del> <ins> objs := []runtime.Object{medPriorityPodInfo.Pod, unschedulablePodInfo.Pod, highPriorityPodInfo.Pod, scheduledPodInfo.Pod} </ins> ctx, cancel := context.WithCancel(context.Background()) defer cancel() q := NewTestQueueWithObjects(ctx, newDefaultQueueSort(), objs)

t.Errorf("Unexpected diff after updating pods (-want, +got):n%s", diff) } <ins> // Attempt to nominate a pod that was deleted from the informer cache. // Nothing should change. q.AddNominatedPod(nonExistentPodInfo, &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominator{}), cmpopts.IgnoreFields(nominator{}, "podLister", "RWMutex")); diff != "" { t.Errorf("Unexpected diff after nominating a deleted pod (-want, +got):n%s", diff) } // Attempt to nominate a pod that was already scheduled in the informer cache. // Nothing should change. scheduledPodCopy := scheduledPodInfo.Pod.DeepCopy() scheduledPodInfo.Pod.Spec.NodeName = "" q.AddNominatedPod(framework.NewPodInfo(scheduledPodCopy), &framework.NominatingInfo{NominatingMode: framework.ModeOverride, NominatedNodeName: "node1"}) if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominator{}), cmpopts.IgnoreFields(nominator{}, "podLister", "RWMutex")); diff != "" { t.Errorf("Unexpected diff after nominating a scheduled pod (-want, +got):n%s", diff) } </ins> // Delete a nominated pod that doesn't have nominatedNodeName in the pod // object. It should be deleted. q.DeleteNominatedPodIfExists(highPriorityPodInfo.Pod)