Change:
<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: Conformance-GCE-master Which test(s) are failing: Since when has it been failing: 11-07 14:41 PST probably after this PR got merged Testgrid link: https://k8s- Reason for failure: Anything else we need to know: /cc /sig scheduling /milestone v1.17 /priority critical-urgent
/assign draveness ahg-g
it is actually after , which was trying to fix WaitForStableCluster is timing out, sadly there are not enough logs from the test to understand why, but likely because there are pods in a namespace other than kube-system from previous tests that never get deleted.
oh I see... so you think it's a flake? Or we need one more fix to the WaitForStableCluster?
We need to fix WaitForStableCluster again. /cc
Thanks for such a quick turnover!

Discussion:
timeout := 10 * time.Minute startTime := time.Now() <del> allPods := getAllPods(c) scheduledSystemPods, currentlyNotScheduledSystemPods := getSystemPods(c) systemPods := scheduledSystemPods + currentlyNotScheduledSystemPods // Wait for system pods to be scheduled, and for pods in all other namespaces to be deleted for currentlyNotScheduledSystemPods != 0 || systemPods != allPods { </del> <ins> // Wait for all pods to be scheduled. allScheduledPods, allNotScheduledPods := getFilteredPods(c, masterNodes, metav1.NamespaceAll) for len(allNotScheduledPods) != 0 { </ins> time.Sleep(2 * time.Second) <del> scheduledSystemPods, currentlyNotScheduledSystemPods := getSystemPods(c) systemPods = scheduledSystemPods + currentlyNotScheduledSystemPods allPods = getAllPods(c) </del> if startTime.Add(timeout).Before(time.Now()) { <ins> framework.Logf("Timed out waiting for the following pods to schedule") for _, p := range allNotScheduledPods { framework.Logf("%v/%v", p.Namespace, p.Name) } </ins> framework.Failf("Timed out after %v waiting for stable cluster.", timeout) break } <ins> allScheduledPods, allNotScheduledPods = getFilteredPods(c, masterNodes, metav1.NamespaceAll) </ins> } <del> return scheduledSystemPods </del> <ins> return len(allScheduledPods) </ins> } <del> // getAllPods lists all pods in the cluster, with succeeded and failed pods filtered out and returns the count func getAllPods(c clientset.Interface) int { allPods, err := c.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}) </del> <ins> // getFilteredPods lists scheduled and not scheduled pods in the given namespace, with succeeded and failed pods filtered out. func getFilteredPods(c clientset.Interface, masterNodes sets.String, ns string) (scheduledPods, notScheduledPods []v1.Pod) { pods, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{}) </ins> framework.ExpectNoError(err, "listing all pods in kube-system namespace while waiting for stable cluster") // API server returns also Pods that succeeded. We need to filter them out. <del> currentPods := make([]v1.Pod, 0, len(allPods.Items)) for _, pod := range allPods.Items { </del> <ins> filteredPods := make([]v1.Pod, 0, len(pods.Items)) for _, pod := range pods.Items { </ins> if pod.Status.Phase != v1.PodSucceeded && pod.Status.Phase != v1.PodFailed { <del> currentPods = append(currentPods, pod) </del> <ins> filteredPods = append(filteredPods, pod) </ins> } } <del> allPods.Items = currentPods return len(allPods.Items) } // getSystemPods lists the pods in the kube-system namespace and returns the number of scheduled and unscheduled pods func getSystemPods(c clientset.Interface) (int, int) { systemPods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(metav1.ListOptions{}) framework.ExpectNoError(err, "listing all pods in kube-system namespace while waiting for stable cluster") scheduledSystemPods, currentlyNotScheduledSystemPods := e2epod.GetPodsScheduled(masterNodes, systemPods) return len(scheduledSystemPods), len(currentlyNotScheduledSystemPods) </del> <ins> pods.Items = filteredPods return e2epod.GetPodsScheduled(masterNodes, pods) </ins> }