Change:
We have an e2e test that involves cluster autoscaler; the test creates two pods back to back which forces CA to scale up the cluster to create two nodes also back to back one for each pod. The two nodes are not created at the same time, there is some delay between them. The problem is that the second pod never gets scheduled. We see in the logs a single scheduling attempt for the second pod that attempted to schedule it on the first node and failed because it didn't fit the node's remaining cpu cores. We don't see any attempts after that for the pod and the test times out waiting for that pod to schedule. What changed in the scheduler: Pre 1.22, on most update/add/delete events to pods/nodes/pvc/storageClass resources, we used to move all unschedulable pods back to the queue. Also every we did force move all unschedulable pods back to the queue indiscriminately. This behavior is not efficient, wasted scheduling cycles and since we re-queue based on last scheduling attempt timestamp, it missed up badly with the best effort FIFO behavior we intend to have for pod scheduling.Since 1.22 a couple of things changed such that unschedulable pods are not placed back into the queue on those events indiscriminately; on each such event, we iterate over unschedulable pods and we place them back into the queue if they pass two checks: the first one happens only on node update/add events, it is carried by the function ; it basically runs the kubelet admission logic (node resource, affinity, taints/tolerations and host port checks) the second check runs on all events (including the node ones); this one is a little more complicated and fine grained: for each unschedulable pod we remember which plugins last failed to fit the pod; at the same time plugins register which events could make an unschedulable pod schedulable (e.g. nodeResources filter plugin registers pod delete and node add/update events). A pod is put back into the queue only if the event matches any of the ones registered by the plugins that last failed the pod. In 1.24 we increased the 60s to (our north star to completely remove this) What went wrong: My theory is that we incorrectly determined that the pod was not eligible to be put back into the queue on events related to the new node; and that the 5min timeout was too long to force moving it back to the queue (I am guessing the whole test timed out), while the 60s was basically masking the problem for 1.22 and 1.23. So what checks are incorrectly failing to move the pod? Here is what happened: Before adding the new node, the pod failed the nodeResource plugin, which only registers for New node was , scheduler receives an add event. The pod was not moved back to the queue because it failed the check on NotReady taint. When the new node became ready, the scheduler receives a node update event, but since it is a taint update event, it didn't match the events that the nodeResources plugin registered for. How to fix it: Since we have the ; pods that previously failed because of NodeResources, NodePorts, TaintsAndTolerations and NodeAffinity (the predicates we already ran in ) should always be placed back into the queue; basically those plugins should register for all node update events. /sig scheduling /cc The second pod to schedule. See above. No response 24 GKE <details</details<details</details<details</details<details</details
/triage accepted

Discussion:
// failed by this plugin schedulable. func (pl *NodeAffinity) EventsToRegister() []framework.ClusterEvent { return []framework.ClusterEvent{ <del> {Resource: framework.Node, ActionType: framework.Add | framework.UpdateNodeLabel}, </del> <ins> {Resource: framework.Node, ActionType: framework.Add | framework.Update}, </ins> } }

// failed by this plugin schedulable. func (pl *NodeName) EventsToRegister() []framework.ClusterEvent { return []framework.ClusterEvent{ <del> {Resource: framework.Node, ActionType: framework.Add}, </del> <ins> {Resource: framework.Node, ActionType: framework.Add | framework.Update}, </ins> } }

return []framework.ClusterEvent{ // Due to immutable fields `spec.containers[*].ports`, pod update events are ignored. {Resource: framework.Pod, ActionType: framework.Delete}, <del> {Resource: framework.Node, ActionType: framework.Add}, </del> <ins> {Resource: framework.Node, ActionType: framework.Add | framework.Update}, </ins> } }

func (f *Fit) EventsToRegister() []framework.ClusterEvent { return []framework.ClusterEvent{ {Resource: framework.Pod, ActionType: framework.Delete}, <del> {Resource: framework.Node, ActionType: framework.Add | framework.UpdateNodeAllocatable}, </del> <ins> {Resource: framework.Node, ActionType: framework.Add | framework.Update}, </ins> } }

// failed by this plugin schedulable. func (pl *TaintToleration) EventsToRegister() []framework.ClusterEvent { return []framework.ClusterEvent{ <del> {Resource: framework.Node, ActionType: framework.Add | framework.UpdateNodeTaint}, </del> <ins> {Resource: framework.Node, ActionType: framework.Add | framework.Update}, </ins> } }

}) } } <ins> func TestNodeEvents(t *testing.T) { // The test verifies that unschedulable pods are re-queued // on node update events. The scenario we are testing is the following: // 1. Create pod1 and node1 that is small enough to only fit pod1; pod1 schedules on node1 // 2. Create pod2, it should be unschedulable due to insufficient cpu // 3. Create node2 with a taint, pod2 should still not schedule // 4. Remove the taint from node2; pod2 should now schedule on node2 testCtx := initTest(t, "node-events") defer testutils.CleanupTest(t, testCtx) defer testCtx.ClientSet.CoreV1().Nodes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) // 1.1 create pod1 pod1, err := createPausePodWithResource(testCtx.ClientSet, "pod1", testCtx.NS.Name, &v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity(80, resource.DecimalSI), }) if err != nil { t.Fatalf("Failed to create pod: %v", err) } // 1.2 Create node1 node1, err := createNode(testCtx.ClientSet, st.MakeNode(). Name("node-events-test-node1"). Capacity(map[v1.ResourceName]string{ v1.ResourcePods:   "32", v1.ResourceCPU:    "100m", v1.ResourceMemory: "30", }).Obj()) if err != nil { t.Fatalf("Failed to create %s: %v", node1.Name, err) } // 1.3 verify pod1 is scheduled err = testutils.WaitForPodToScheduleWithTimeout(testCtx.ClientSet, pod1, time.Second*5) if err != nil { t.Errorf("Pod %s didn't schedule: %v", pod1.Name, err) } // 2. create pod2 pod2, err := createPausePodWithResource(testCtx.ClientSet, "pod2", testCtx.NS.Name, &v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity(40, resource.DecimalSI), }) if err != nil { t.Fatalf("Failed to create pod %v: %v", pod2.Name, err) } if err := waitForPodUnschedulable(testCtx.ClientSet, pod2); err != nil { t.Errorf("Pod %v got scheduled: %v", pod2.Name, err) } // 3.1 Create node2 with a taint node2 := st.MakeNode(). Name("node-events-test-node2"). Capacity(map[v1.ResourceName]string{ v1.ResourcePods:   "32", v1.ResourceCPU:    "100m", v1.ResourceMemory: "30", }). Label("affinity-key", "affinity-value"). Taints([]v1.Taint{{Key: "taint-key", Effect: v1.TaintEffectNoSchedule}}).Obj() node2, err = createNode(testCtx.ClientSet, node2) if err != nil { t.Fatalf("Failed to create %s: %v", node2.Name, err) } // make sure the scheduler received the node add event by creating a pod that only fits node2 plugPod := st.MakePod().Name("plug-pod").Namespace(testCtx.NS.Name).Container("pause"). Req(map[v1.ResourceName]string{v1.ResourceCPU: "40m"}). NodeAffinityIn("affinity-key", []string{"affinity-value"}). Toleration("taint-key").Obj() plugPod, err = testCtx.ClientSet.CoreV1().Pods(plugPod.Namespace).Create(context.TODO(), plugPod, metav1.CreateOptions{}) if err != nil { t.Fatalf("Failed to create pod %v: %v", plugPod.Name, err) } err = testutils.WaitForPodToScheduleWithTimeout(testCtx.ClientSet, plugPod, time.Second*5) if err != nil { t.Errorf("Pod %s didn't schedule: %v", plugPod.Name, err) } // 3.2 pod2 still unschedulable if err := waitForPodUnschedulable(testCtx.ClientSet, pod2); err != nil { t.Errorf("Pod %v got scheduled: %v", pod2.Name, err) } // 4. Remove node taint, pod2 should schedule node2.Spec.Taints = nil node2, err = updateNode(testCtx.ClientSet, node2) if err != nil { t.Fatalf("Failed to update %s: %v", node2.Name, err) } err = testutils.WaitForPodToScheduleWithTimeout(testCtx.ClientSet, pod2, time.Second*5) if err != nil { t.Errorf("Pod %s didn't schedule: %v", pod2.Name, err) } } </ins>

return err } <ins> func updateNode(cs clientset.Interface, node *v1.Node) (*v1.Node, error) { return cs.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) } </ins> func createNode(cs clientset.Interface, node *v1.Node) (*v1.Node, error) { return cs.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}) }

} _, cond := podutil.GetPodCondition(&pod.Status, v1.PodScheduled) return cond != nil && cond.Status == v1.ConditionFalse && <del> cond.Reason == v1.PodReasonUnschedulable, nil </del> <ins> cond.Reason == v1.PodReasonUnschedulable && pod.Spec.NodeName == "", nil </ins> } }