Change:
<!-- Please only use this template for submitting enhancement requests --What would you like to be : Today, the scheduler re-processes unschedulable nodes . Node updates are sent by Kubelet frequently (every 10 seconds) and often times they don't change any of the node properties that could potentially make them feasible. We should be smarter about these updates and move pods to the schedulable queue only when certain properties of nodes change that could make them feasible. These properties include node allocatable, node conditions, taints, labels, .... Why is this needed: Improves performance of the scheduler and avoids doing unnecessary attempts to schedule pods. This is particularly helpful when a cluster has a lot of unschedulable pods. <!-- DO NOT EDIT BELOW THIS LINE --/kind feature This is not a feature really. It is performance optimization, but in the absence of such a label, I need to use "feature" here. /sig scheduling
I can help to fix it. /assign mlmhl
The main reason for this problem is / update their status too frequently and most of these updates are heartbeats without any meaningful change. Recently sig-node introduces a new to alleviate this problem. With this mechanism, kubelet will split heartbeat and node status updates into two independent operations, and reduce the frequency of status updates to once per 1 minute(or more). In the long run this mechanism can indeed alleviate our problems, but at present it does not help us much as this is still an alpha feature and the node status updates frequency is still a bit high(per 1 minute right now).
Maybe we can try comparing old and new nodes after stripping their heartbeat info, like this:
I think that had sent earlier is more efficient. However, that PR does not filter out . In Wei's code, the heartbeat time is removed correctly. I think we should fix this issue and resubmit that PR.
could you resubmit a PR to: disregard in conditions comparison: and (optionally) disregard timestamp in taints comparison:
Thanks for all your advice, a new PR is submitted: This PR does not include your second suggestion(disregard timestamp in taints comparison) as I'm not sure if can be changed when node updating. If it can be changed, I will add this filter later, otherwise I think we can keep the current implementation to make the compare operation simpler and faster. What's more, just as I said in the PR's description, I'm not sure how much side effects this change has on scheduler's resource consumption, as it always copy node's conditions before comparison.
I am thinking whether we could move this logic(whether node became more schedulable) to kubelet, kubelet reports it through heartbeats.

Discussion:
embed = [":go_default_library"], deps = [ "//pkg/api/testing:go_default_library", <ins> "//pkg/apis/core:go_default_library", </ins> "//pkg/scheduler/algorithm:go_default_library", "//pkg/scheduler/algorithm/priorities:go_default_library", "//pkg/scheduler/api:go_default_library",

"//pkg/scheduler/testing:go_default_library", "//pkg/scheduler/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",

} c.invalidateCachedPredicatesOnNodeUpdate(newNode, oldNode) <del> c.podQueue.MoveAllToActiveQueue() </del> <ins> // Only activate unschedulable pods if the node became more schedulable. if nodeSchedulingPropertiesChanged(newNode, oldNode) { c.podQueue.MoveAllToActiveQueue() } </ins> } func (c *configFactory) invalidateCachedPredicatesOnNodeUpdate(newNode *v1.Node, oldNode *v1.Node) {

} } <ins> func nodeSchedulingPropertiesChanged(newNode *v1.Node, oldNode *v1.Node) bool { if nodeAllocatableChanged(newNode, oldNode) { glog.V(4).Infof("Allocatable resource of node %s changed", newNode.Name) return true } if nodeLabelsChanged(newNode, oldNode) { glog.V(4).Infof("Labels of node %s changed", newNode.Name) return true } if nodeTaintsChanged(newNode, oldNode) { glog.V(4).Infof("Taints of node %s changed", newNode.Name) return true } if nodeConditionsChanged(newNode, oldNode) { glog.V(4).Infof("Conditions of node %s changed", newNode.Name) return true } if newNode.Spec.Unschedulable != oldNode.Spec.Unschedulable && newNode.Spec.Unschedulable == false { glog.V(4).Infof("Node %s changed to schedulable", newNode.Name) return true } return false } func nodeAllocatableChanged(newNode *v1.Node, oldNode *v1.Node) bool { return !reflect.DeepEqual(oldNode.Status.Allocatable, newNode.Status.Allocatable) } func nodeLabelsChanged(newNode *v1.Node, oldNode *v1.Node) bool { return !reflect.DeepEqual(oldNode.GetLabels(), newNode.GetLabels()) } func nodeTaintsChanged(newNode *v1.Node, oldNode *v1.Node) bool { if !reflect.DeepEqual(newNode.Spec.Taints, oldNode.Spec.Taints) { return true } oldTaints, oldErr := helper.GetTaintsFromNodeAnnotations(oldNode.GetAnnotations()) if oldErr != nil { // If parse old node's taint annotation failed, we assume node's taint changed. glog.Errorf("Failed to get taints from annotation of old node %s: %v", oldNode.Name, oldErr) return true } newTaints, newErr := helper.GetTaintsFromNodeAnnotations(newNode.GetAnnotations()) if newErr != nil { // If parse new node's taint annotation failed, we assume node's taint changed. glog.Errorf("Failed to get taints from annotation of new node %s: %v", newNode.Name, newErr) return true } if !reflect.DeepEqual(oldTaints, newTaints) { return true } return false } func nodeConditionsChanged(newNode *v1.Node, oldNode *v1.Node) bool { return !reflect.DeepEqual(oldNode.Status.Conditions, newNode.Status.Conditions) } </ins> func (c *configFactory) deleteNodeFromCache(obj interface{}) { var node *v1.Node switch t := obj.(type) {

"time" "k8s.io/api/core/v1" <ins> "k8s.io/apimachinery/pkg/api/resource" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets"

clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" apitesting "k8s.io/kubernetes/pkg/api/testing" <ins> "k8s.io/kubernetes/pkg/apis/core" </ins> "k8s.io/kubernetes/pkg/scheduler/algorithm" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" latestschedulerapi "k8s.io/kubernetes/pkg/scheduler/api/latest"

t.Errorf("Expected binder %q but got %q", expectedBinderType, binderType) } } <ins> func TestNodeAllocatableChanged(t *testing.T) { newQuantity := func(value int64) resource.Quantity { return *resource.NewQuantity(value, resource.BinarySI) } for _, c := range []struct { Changed        bool OldAllocatable v1.ResourceList NewAllocatable v1.ResourceList }{ // No allocatable resources changed. { Changed:        false, OldAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024)}, NewAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024)}, }, // New node has more allocatable resources. { Changed:        true, OldAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024)}, NewAllocatable: v1.ResourceList{v1.ResourceMemory: newQuantity(1024), v1.ResourceStorage: newQuantity(1024)}, }, } { oldNode := &v1.Node{Status: v1.NodeStatus{Allocatable: c.OldAllocatable}} newNode := &v1.Node{Status: v1.NodeStatus{Allocatable: c.NewAllocatable}} changed := nodeAllocatableChanged(newNode, oldNode) if changed != c.Changed { t.Errorf("nodeAllocatableChanged should be %t, got %t", c.Changed, changed) } } } func TestNodeLabelsChanged(t *testing.T) { for _, c := range []struct { Changed   bool OldLabels map[string]string NewLabels map[string]string }{ // No labels changed. {Changed: false, OldLabels: map[string]string{"foo": "bar"}, NewLabels: map[string]string{"foo": "bar"}}, // Labels changed. {Changed: true, OldLabels: map[string]string{"foo": "bar"}, NewLabels: map[string]string{"foo": "bar", "test": "value"}}, } { oldNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: c.OldLabels}} newNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Labels: c.NewLabels}} changed := nodeLabelsChanged(newNode, oldNode) if changed != c.Changed { t.Errorf("nodeLabelsChanged should be %t, got %t", c.Changed, changed) } } } func TestNodeTaintsChanged(t *testing.T) { for _, c := range []struct { Changed        bool OldTaints      []v1.Taint NewTaints      []v1.Taint OldAnnotations map[string]string NewAnnotations map[string]string }{ // Taints use annotation and no change. { Changed:        false, OldAnnotations: map[string]string{core.TaintsAnnotationKey: `[{"key":"value"}]`}, NewAnnotations: map[string]string{core.TaintsAnnotationKey: `[{"key":"value"}]`}, }, // Taints use annotation and changed. { Changed:        true, OldAnnotations: map[string]string{core.TaintsAnnotationKey: `[{"key":"value1"}]`}, NewAnnotations: map[string]string{core.TaintsAnnotationKey: `[{"key":"value2"}]`}, }, // Taints use Spec.Taints and no change. { Changed:   false, OldTaints: []v1.Taint{{Key: "key", Value: "value"}}, NewTaints: []v1.Taint{{Key: "key", Value: "value"}}, }, // Taints use Spec.Taints and changed. { Changed:   true, OldTaints: []v1.Taint{{Key: "key", Value: "value1"}}, NewTaints: []v1.Taint{{Key: "key", Value: "value2"}}, }, } { oldNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Annotations: c.OldAnnotations}, Spec: v1.NodeSpec{Taints: c.OldTaints}} newNode := &v1.Node{ObjectMeta: metav1.ObjectMeta{Annotations: c.NewAnnotations}, Spec: v1.NodeSpec{Taints: c.NewTaints}} changed := nodeTaintsChanged(newNode, oldNode) if changed != c.Changed { t.Errorf("nodeTaintsChanged should be %t, not %t", c.Changed, changed) } } } func TestNodeConditionsChanged(t *testing.T) { for _, c := range []struct { Changed       bool OldConditions []v1.NodeCondition NewConditions []v1.NodeCondition }{ // No conditions changed. { Changed:       false, OldConditions: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}}, NewConditions: []v1.NodeCondition{{Type: v1.NodeOutOfDisk, Status: v1.ConditionTrue}}, }, // New node has more healthy conditions. { Changed:       true, OldConditions: []v1.NodeCondition{}, NewConditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}, }, // NodeReady False -> True { Changed:       true, OldConditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionFalse}}, NewConditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}, }, } { oldNode := &v1.Node{Status: v1.NodeStatus{Conditions: c.OldConditions}} newNode := &v1.Node{Status: v1.NodeStatus{Conditions: c.NewConditions}} changed := nodeConditionsChanged(newNode, oldNode) if changed != c.Changed { t.Errorf("nodeConditionsChanged should be %t, got %t", c.Changed, changed) } } } </ins>