Change:
<!-- Please only use this template for submitting enhancement requests --What would you like to be : Today, the scheduler re-processes unschedulable nodes . Node updates are sent by Kubelet frequently (every 10 seconds) and often times they don't change any of the node properties that could potentially make them feasible. We should be smarter about these updates and move pods to the schedulable queue only when certain properties of nodes change that could make them feasible. These properties include node allocatable, node conditions, taints, labels, .... Why is this needed: Improves performance of the scheduler and avoids doing unnecessary attempts to schedule pods. This is particularly helpful when a cluster has a lot of unschedulable pods. <!-- DO NOT EDIT BELOW THIS LINE --/kind feature This is not a feature really. It is performance optimization, but in the absence of such a label, I need to use "feature" here. /sig scheduling
I can help to fix it. /assign mlmhl
The main reason for this problem is / update their status too frequently and most of these updates are heartbeats without any meaningful change. Recently sig-node introduces a new to alleviate this problem. With this mechanism, kubelet will split heartbeat and node status updates into two independent operations, and reduce the frequency of status updates to once per 1 minute(or more). In the long run this mechanism can indeed alleviate our problems, but at present it does not help us much as this is still an alpha feature and the node status updates frequency is still a bit high(per 1 minute right now).
Maybe we can try comparing old and new nodes after stripping their heartbeat info, like this:
I think that had sent earlier is more efficient. However, that PR does not filter out . In Wei's code, the heartbeat time is removed correctly. I think we should fix this issue and resubmit that PR.
could you resubmit a PR to: disregard in conditions comparison: and (optionally) disregard timestamp in taints comparison:
Thanks for all your advice, a new PR is submitted: This PR does not include your second suggestion(disregard timestamp in taints comparison) as I'm not sure if can be changed when node updating. If it can be changed, I will add this filter later, otherwise I think we can keep the current implementation to make the compare operation simpler and faster. What's more, just as I said in the PR's description, I'm not sure how much side effects this change has on scheduler's resource consumption, as it always copy node's conditions before comparison.
I am thinking whether we could move this logic(whether node became more schedulable) to kubelet, kubelet reports it through heartbeats.

Discussion:
"//pkg/scheduler/internal/queue:go_default_library", "//pkg/scheduler/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",

} c.invalidateCachedPredicatesOnNodeUpdate(newNode, oldNode) <del> c.podQueue.MoveAllToActiveQueue() </del> <ins> // Only activate unschedulable pods if the node became more schedulable. // We skip the node property comparison when there is no unschedulable pods in the queue // to save processing cycles. We still trigger a move to active queue to cover the case // that a pod being processed by the scheduler is determined unschedulable. We want this // pod to be reevaluated when a change in the cluster happens. if c.podQueue.NumUnschedulablePods() == 0 || nodeSchedulingPropertiesChanged(newNode, oldNode) { c.podQueue.MoveAllToActiveQueue() } </ins> } func (c *configFactory) invalidateCachedPredicatesOnNodeUpdate(newNode *v1.Node, oldNode *v1.Node) {

} } <ins> func nodeSchedulingPropertiesChanged(newNode *v1.Node, oldNode *v1.Node) bool { if nodeSpecUnschedulableChanged(newNode, oldNode) { return true } if nodeAllocatableChanged(newNode, oldNode) { return true } if nodeLabelsChanged(newNode, oldNode) { return true } if nodeTaintsChanged(newNode, oldNode) { return true } if nodeConditionsChanged(newNode, oldNode) { return true } return false } func nodeAllocatableChanged(newNode *v1.Node, oldNode *v1.Node) bool { return !reflect.DeepEqual(oldNode.Status.Allocatable, newNode.Status.Allocatable) } func nodeLabelsChanged(newNode *v1.Node, oldNode *v1.Node) bool { return !reflect.DeepEqual(oldNode.GetLabels(), newNode.GetLabels()) } func nodeTaintsChanged(newNode *v1.Node, oldNode *v1.Node) bool { return !reflect.DeepEqual(newNode.Spec.Taints, oldNode.Spec.Taints) } func nodeConditionsChanged(newNode *v1.Node, oldNode *v1.Node) bool { strip := func(conditions []v1.NodeCondition) map[v1.NodeConditionType]v1.ConditionStatus { conditionStatuses := make(map[v1.NodeConditionType]v1.ConditionStatus, len(conditions)) for i := range conditions { conditionStatuses[conditions[i].Type] = conditions[i].Status } return conditionStatuses } return !reflect.DeepEqual(strip(oldNode.Status.Conditions), strip(newNode.Status.Conditions)) } func nodeSpecUnschedulableChanged(newNode *v1.Node, oldNode *v1.Node) bool { return newNode.Spec.Unschedulable != oldNode.Spec.Unschedulable && newNode.Spec.Unschedulable == false } </ins> func (c *configFactory) deleteNodeFromCache(obj interface{}) { var node *v1.Node switch t := obj.(type) {

"time" "k8s.io/api/core/v1" <ins> "k8s.io/apimachinery/pkg/api/resource" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets"

t.Errorf("Expected binder %q but got %q", expectedBinderType, binderType) } } <ins> func TestNodeAllocatableChanged(t *testing.T) { newQuantity := func(value int64) resource.Quantity { return *resource.NewQuantity(value, resource.BinarySI) } for _, c := range []struct { Name           string Changed        bool OldAllocatable v1.ResourceList NewAllocatable v1.ResourceList }{ { Name:           "no allocatable resources changed", Changed:        false, OldAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024)}, NewAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024)}, }, { Name:           "new node has more allocatable resources", Changed:        true, OldAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024)}, NewAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024), v1.ResourceStorage: newQuantity(1024)}, }, } { oldNode := &v1.Node{Status: v1.NodeStatus{Allocatable: c.OldAllocatable}} newNode := &v1.Node{Status: v1.NodeStatus{Allocatable: c.NewAllocatable}} changed := nodeAllocatableChanged(newNode, oldNode) if changed != c.Changed { t.Errorf("nodeAllocatableChanged should be %t, got %t", c.Changed, changed) } } } func TestNodeLabelsChanged(t *testing.T) { for _, c := range []struct { Name      string Changed   bool OldLabels map[string]string NewLabels map[string]string }{ { Name:      "no labels changed", Changed:   false, OldLabels: map[string]string{"foo": "bar"}, NewLabels: map[string]string{"foo": "bar"}, }, // Labels changed. { Name:      "new node has more labels", Changed:   true, OldLabels: map[string]string{"foo": "bar"}, NewLabels: map[string]string{"foo": "bar", "test": "value"}, }, } { oldNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: c.OldLabels}} newNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: c.NewLabels}} changed := nodeLabelsChanged(newNode, oldNode) if changed != c.Changed { t.Errorf("Test case %q failed: should be %t, got %t", c.Name, c.Changed, changed) } } } func TestNodeTaintsChanged(t *testing.T) { for _, c := range []struct { Name      string Changed   bool OldTaints []v1.Taint NewTaints []v1.Taint }{ { Name:      "no taint changed", Changed:   false, OldTaints: []v1.Taint{{Key: "key", Value: "value"}}, NewTaints: []v1.Taint{{Key: "key", Value: "value"}}, }, { Name:      "taint value changed", Changed:   true, OldTaints: []v1.Taint{{Key: "key", Value: "value1"}}, NewTaints: []v1.Taint{{Key: "key", Value: "value2"}}, }, } { oldNode := &v1.Node{Spec: v1.NodeSpec{Taints: c.OldTaints}} newNode := &v1.Node{Spec: v1.NodeSpec{Taints: c.NewTaints}} changed := nodeTaintsChanged(newNode, oldNode) if changed != c.Changed { t.Errorf("Test case %q failed: should be %t, not %t", c.Name, c.Changed, changed) } } } func TestNodeConditionsChanged(t *testing.T) { nodeConditionType := reflect.TypeOf(v1.NodeCondition{}) if nodeConditionType.NumField() != 6 { t.Errorf("NodeCondition type has changed. The nodeConditionsChanged() function must be reevaluated.") } for _, c := range []struct { Name          string Changed       bool OldConditions []v1.NodeCondition NewConditions []v1.NodeCondition }{ { Name:          "no condition changed", Changed:       false, OldConditions: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}}, NewConditions: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}}, }, { Name:          "only LastHeartbeatTime changed", Changed:       false, OldConditions: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue, LastHeartbeatTime: metav1.Unix(1, 0)}}, NewConditions: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue, LastHeartbeatTime: metav1.Unix(2, 0)}}, }, { Name:          "new node has more healthy conditions", Changed:       true, OldConditions: []v1.NodeCondition{}, NewConditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}, }, { Name:          "new node has less unhealthy conditions", Changed:       true, OldConditions: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}}, NewConditions: []v1.NodeCondition{}, }, { Name:          "condition status changed", Changed:       true, OldConditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionFalse}}, NewConditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}, }, } { oldNode := &v1.Node{Status: v1.NodeStatus{Conditions: c.OldConditions}} newNode := &v1.Node{Status: v1.NodeStatus{Conditions: c.NewConditions}} changed := nodeConditionsChanged(newNode, oldNode) if changed != c.Changed { t.Errorf("Test case %q failed: should be %t, got %t", c.Name, c.Changed, changed) } } } </ins>

Close() // DeleteNominatedPodIfExists deletes nominatedPod from internal cache DeleteNominatedPodIfExists(pod *v1.Pod) <ins> // NumUnschedulablePods returns the number of unschedulable pods exist in the SchedulingQueue. NumUnschedulablePods() int </ins> } // NewSchedulingQueue initializes a new scheduling queue. If pod priority is

// DeleteNominatedPodIfExists does nothing in FIFO. func (f *FIFO) DeleteNominatedPodIfExists(pod *v1.Pod) {} <ins> // NumUnschedulablePods returns the number of unschedulable pods exist in the SchedulingQueue. func (f *FIFO) NumUnschedulablePods() int { return 0 } </ins> // NewFIFO creates a FIFO object. func NewFIFO() *FIFO { return &FIFO{FIFO: cache.NewFIFO(cache.MetaNamespaceKeyFunc)}

return bo1.Before(bo2) } <ins> // NumUnschedulablePods returns the number of unschedulable pods exist in the SchedulingQueue. func (p *PriorityQueue) NumUnschedulablePods() int { p.lock.RLock() defer p.lock.RUnlock() return len(p.unschedulableQ.pods) } </ins> // UnschedulablePodsMap holds pods that cannot be scheduled. This data structure // is used to implement unschedulableQ. type UnschedulablePodsMap struct {