Change:
framework.Failf("Timed out after %v waiting for stable cluster.", timeout) break } <del> allScheduledPods, allNotScheduledPods = getFilteredPods(c, masterNodes, metav1.NamespaceAll) </del> <ins> allScheduledPods, allNotScheduledPods = getScheduledAndUnscheduledPods(c, masterNodes, metav1.NamespaceAll) </ins> } return len(allScheduledPods) } <del> // getFilteredPods lists scheduled and not scheduled pods in the given namespace, with succeeded and failed pods filtered out. func getFilteredPods(c clientset.Interface, masterNodes sets.String, ns string) (scheduledPods, notScheduledPods []v1.Pod) { </del> <ins> // WaitForPodsToBeDeleted waits until pods that are terminating to get deleted. func WaitForPodsToBeDeleted(c clientset.Interface) { startTime := time.Now() deleting := getDeletingPods(c, metav1.NamespaceAll) for len(deleting) != 0 { if startTime.Add(timeout).Before(time.Now()) { framework.Logf("Pods still not deleted") for _, p := range deleting { framework.Logf("%v/%v", p.Namespace, p.Name) } framework.Failf("Timed out after %v waiting for pods to be deleted", timeout) break } time.Sleep(waitTime) deleting = getDeletingPods(c, metav1.NamespaceAll) } } // getScheduledAndUnscheduledPods lists scheduled and not scheduled pods in the given namespace, with succeeded and failed pods filtered out. func getScheduledAndUnscheduledPods(c clientset.Interface, masterNodes sets.String, ns string) (scheduledPods, notScheduledPods []v1.Pod) { </ins> pods, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{}) <del> framework.ExpectNoError(err, "listing all pods in kube-system namespace while waiting for stable cluster") </del> <ins> framework.ExpectNoError(err, fmt.Sprintf("listing all pods in namespace %q while waiting for stable cluster", ns)) </ins> // API server returns also Pods that succeeded. We need to filter them out. filteredPods := make([]v1.Pod, 0, len(pods.Items)) <del> for _, pod := range pods.Items { if pod.Status.Phase != v1.PodSucceeded && pod.Status.Phase != v1.PodFailed { filteredPods = append(filteredPods, pod) </del> <ins> for _, p := range pods.Items { if !podTerminated(p) { filteredPods = append(filteredPods, p) </ins> } } pods.Items = filteredPods return e2epod.GetPodsScheduled(masterNodes, pods) } <ins> // getDeletingPods returns whether there are any pods marked for deletion. func getDeletingPods(c clientset.Interface, ns string) []v1.Pod { pods, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{}) framework.ExpectNoError(err, fmt.Sprintf("listing all pods in namespace %q while waiting for pods to terminate", ns)) var deleting []v1.Pod for _, p := range pods.Items { if p.ObjectMeta.DeletionTimestamp != nil && !podTerminated(p) { deleting = append(deleting, p) } } return deleting } func podTerminated(p v1.Pod) bool { return p.Status.Phase == v1.PodSucceeded || p.Status.Phase == v1.PodFailed } </ins>

Discussion:
<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: soak-gce-gci-1.13, others Which test(s) are failing: [sig-scheduling] SchedulerPredicates [Serial] validates MaxPods limit number of pods that are allowed to run [Slow] Since when has it been failing: Rougly forever. shows how this test was flaking back during the 1.11 cycle. There's been no sudden increase in frequency, but a test that flakes roughly 1.5 times per hour needs looking into, and probably tweaking. Testgrid link: https://k8s- Reason for failure: Most of the time, these flakes are timeouts. However, sometimes they have other causes; see Triage link: Anything else we need to know: This is part of a general effort to clean up our CI signal by eliminating longstanding, frequent flakes. <!-- DO NOT EDIT BELOW THIS LINE --/kind flake /priority important-longterm /sig scheduling attn:
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale /help-wanted
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
There are several different failures in this test, I was in particular looking at which usually boils down to something like this My hunch is that we are trying to schedule more than we should. This is how we compute how many pods to schedule which just takes max capacity and substracts whatever this returns My guess would be that the computation is not completely correct. My idea would be to add more logging into to be able to figure out more. Does that sound reasonable to you or would you take another approach? cc.

I went over all the pods that were on the node before the test started and then after and diff-ed all the pods that are not started by the test "grep -v maxp-" and did not found any pod that would start in the middle of test and was not present in the beginning :/ but there were 18 pods in the beginning, 330 is the full capacity so we should have started 312 to fully saturate, while we were trying to start 313. (maybe some pod restarted right when we were starting the test?) I am looking at this latest case but there's many like this one
Can we possibly be missing some pods here? pods with no node assigned that are not unschedulable?
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
This just got much worse in the last weeks, I tried looking into PRs along the time and hit into this could someone please verify it that could have made things worse? I'll also try to look tomorrow since I investigated this while ago :)
Anyone can help here? It's constantly red for several days, it's not even flaking anymore in our release dashboards /priority important-soon