Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: /kind bug What you expected to happen: Problem 1: Predicates filter nodes. Existing pods on those nodes will be not be counted when calculating max pods per (zone or node) resulting in imbalanced cluster. Eg: removes nodes which cannot . If any of the pods are already scheduled on this node, they are not considered when counting in . Problem 2: When there are 2 selectors(service and replication controller), it is sufficient to match any one selector for distribution. This creates imbalance ]. Pods from previous deploys matches and are counted when distributing pods across zones/nodes (Even though they do not match ) . These pods will be deleted. After the deploy completes, the cluster is imbalanced - by zone and/or pods per node. How to reproduce it (as minimally and precisely as possible): Problem 1: Get pods to get scheduled on nodes with very high cpu utilization [Existing cpu utilization + 1 new pod will result in almost equal to 0]. GeneralPredicate will drop this node after scheduling the first pod. This will lead to over utilization of an already loaded zone. If there are enough nodes like this one, it will create a pile on effect resulting in most pods scheduled in an already loaded zone. Problem 2: Have 2 selectors and deploy 30 pods across 3 zones. Twice. Anything else we need to know?: Environment: Kubernetes version (use ): /sig scheduling <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
re Problem 1: for per node, we only need to compare current nodes as they will share the same maxPods. for per zone, we need to account other nodes, e.g. because of resource predicates. re problem 2: a little about your case :(
Problem 1: : It does not matter if the denominator is all nodes or current nodes. One with higher utilization will still end up with a lower score. : It is critical that the denominator is all nodes. Else we are ending up with highly imbalanced cluster. Problem 2: Here is an example list of selectors. All pods matching any one service selector are counted in So in our case, old pods are counted during scheduling. But they get killed during the deployment. If the maxSurge and maxUnavailable is high enough, you will end up with an imbalanced zone after deployment.
Gentle ping.
/assign bsalamat
Re: Problem 1 - You can also reproduce this by using pod anti-affinity. Zone score is calculated only based on nodes filtered by predicates. IMO zone score should be calculated using all nodes.
I will try to separate this into two PRs. Any other suggestions welcome.
I am working Problem 1. Looks like I have to open a new issue.

Discussion:
}, nil } <del> count := int(0) for _, nodePod := range nodeInfo.Pods() { if pod.Namespace != nodePod.Namespace { continue } // When we are replacing a failed pod, we often see the previous // deleted version while scheduling the replacement. // Ignore the previous deleted version for spreading purposes // (it can still be considered for resource restrictions etc.) if nodePod.DeletionTimestamp != nil { klog.V(4).Infof("skipping pending-deleted pod: %s/%s", nodePod.Namespace, nodePod.Name) continue } for _, selector := range selectors { if selector.Matches(labels.Set(nodePod.ObjectMeta.Labels)) { count++ break } } } </del> <ins> count := countMatchingPods(pod.Namespace, selectors, nodeInfo) </ins> return schedulerapi.HostPriority{ Host:  node.Name, <del> Score: int(count), </del> <ins> Score: count, </ins> }, nil }

return labeledNodes, nonLabeledNodes } <del> // filteredPod get pods based on namespace and selector func filteredPod(namespace string, selector labels.Selector, nodeInfo *schedulernodeinfo.NodeInfo) (pods []*v1.Pod) { if nodeInfo.Pods() == nil || len(nodeInfo.Pods()) == 0 || selector == nil { return []*v1.Pod{} </del> <ins> // countMatchingPods cout pods based on namespace and matching all selectors func countMatchingPods(namespace string, selectors []labels.Selector, nodeInfo *schedulernodeinfo.NodeInfo) int { if nodeInfo.Pods() == nil || len(nodeInfo.Pods()) == 0 || len(selectors) == 0 { return 0 </ins> } <ins> count := 0 </ins> for _, pod := range nodeInfo.Pods() { // Ignore pods being deleted for spreading purposes // Similar to how it is done for SelectorSpreadPriority <del> if namespace == pod.Namespace && pod.DeletionTimestamp == nil && selector.Matches(labels.Set(pod.Labels)) { pods = append(pods, pod) </del> <ins> if namespace == pod.Namespace && pod.DeletionTimestamp == nil { matches := true for _, selector := range selectors { if !selector.Matches(labels.Set(pod.Labels)) { matches = false break } } if matches { count++ } </ins> } } <del> return </del> <ins> return count </ins> } // CalculateAntiAffinityPriorityMap spreads pods by minimizing the number of pods belonging to the same service

firstServiceSelector = getFirstServiceSelector(pod, s.serviceLister) } //pods matched namespace,selector on current node <del> matchedPodsOfNode := filteredPod(pod.Namespace, firstServiceSelector, nodeInfo) </del> <ins> var selectors []labels.Selector if firstServiceSelector != nil { selectors = append(selectors, firstServiceSelector) } score := countMatchingPods(pod.Namespace, selectors, nodeInfo) </ins> return schedulerapi.HostPriority{ Host:  node.Name, <del> Score: int(len(matchedPodsOfNode)), </del> <ins> Score: score, </ins> }, nil }

rcs:      []*v1.ReplicationController{{Spec: v1.ReplicationControllerSpec{Selector: map[string]string{"foo": "bar"}}}}, services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: map[string]string{"baz": "blah"}}}}, // "baz=blah" matches both labels1 and labels2, and "foo=bar" matches only labels 1. This means that we assume that we want to <del> // do spreading between all pods. The result should be exactly as above. expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 5}}, </del> <ins> // do spreading pod2 and pod3 and not pod1. expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 0}}, </ins> name:         "service with partial pod label matches with service and replication controller", }, {

services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: map[string]string{"baz": "blah"}}}}, rss:      []*apps.ReplicaSet{{Spec: apps.ReplicaSetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}}}}, // We use ReplicaSet, instead of ReplicationController. The result should be exactly as above. <del> expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 5}}, </del> <ins> expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 0}}, </ins> name:         "service with partial pod label matches with service and replica set", }, {

nodes:        []string{"machine1", "machine2"}, services:     []*v1.Service{{Spec: v1.ServiceSpec{Selector: map[string]string{"baz": "blah"}}}}, sss:          []*apps.StatefulSet{{Spec: apps.StatefulSetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}}}}, <del> expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 5}}, name:         "service with partial pod label matches with service and replica set", </del> <ins> expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 0}}, name:         "service with partial pod label matches with service and stateful set", </ins> }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"foo": "bar", "bar": "foo"}, OwnerReferences: controllerRef("ReplicationController", "name", "abc123")}},

nodes:    []string{"machine1", "machine2"}, rcs:      []*v1.ReplicationController{{Spec: v1.ReplicationControllerSpec{Selector: map[string]string{"foo": "bar"}}}}, services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: map[string]string{"bar": "foo"}}}}, <del> // Taken together Service and Replication Controller should match all Pods, hence result should be equal to one above. expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 5}}, name:         "disjoined service and replication controller should be treated equally", </del> <ins> // Taken together Service and Replication Controller should match no pods. expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 10}, {Host: "machine2", Score: 10}}, name:         "disjoined service and replication controller matches no pods", </ins> }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"foo": "bar", "bar": "foo"}, OwnerReferences: controllerRef("ReplicaSet", "name", "abc123")}},

services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: map[string]string{"bar": "foo"}}}}, rss:      []*apps.ReplicaSet{{Spec: apps.ReplicaSetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}}}}, // We use ReplicaSet, instead of ReplicationController. The result should be exactly as above. <del> expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 5}}, name:         "disjoined service and replica set should be treated equally", </del> <ins> expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 10}, {Host: "machine2", Score: 10}}, name:         "disjoined service and replica set matches no pods", </ins> }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"foo": "bar", "bar": "foo"}, OwnerReferences: controllerRef("StatefulSet", "name", "abc123")}},

nodes:        []string{"machine1", "machine2"}, services:     []*v1.Service{{Spec: v1.ServiceSpec{Selector: map[string]string{"bar": "foo"}}}}, sss:          []*apps.StatefulSet{{Spec: apps.StatefulSetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}}}}, <del> expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 0}, {Host: "machine2", Score: 5}}, name:         "disjoined service and replica set should be treated equally", </del> <ins> expectedList: []schedulerapi.HostPriority{{Host: "machine1", Score: 10}, {Host: "machine2", Score: 10}}, name:         "disjoined service and stateful set matches no pods", </ins> }, { pod: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: labels1, OwnerReferences: controllerRef("ReplicationController", "name", "abc123")}},