Change:
From After , Multi-AZ Clusters tests in 1.19 and 1.20 started timing out as createBalancedPodForNodes is extremely inefficient in scale: for each node (out of 5k), it lists all pods in a cluster serially (each list takes ~5s), which translates to ~7 hours. Optimizing this function by listing pods once and then in memory by Spec.NodeName /kind bug /priority critical-urgent /sig scheduling
/triage accepted
/assign
/reopen Looks that ComputeCPUMemFraction took only 12 minutes, but it stuck later, on It is scheduling one pod per each node: Which is waiting 5s to be ready: 5k nodes * 5s = ~7h
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign Can you do the pod creations in parallel?
Yes, good idea. I'll update to do that

Discussion:
// We check for double because it needs to support at least the cri-o minimum // plus whatever delta between node usages (which could be up to or at least crioMinMemLimit) func nodesAreTooUtilized(cs clientset.Interface, nodeList *v1.NodeList) bool { <ins> nodeNameToPodList := podListForEachNode(cs) </ins> for _, node := range nodeList.Items { <del> _, memFraction, _, memAllocatable := computeCPUMemFraction(cs, node, podRequestedResource) </del> <ins> _, memFraction, _, memAllocatable := computeCPUMemFraction(node, podRequestedResource, nodeNameToPodList[node.Name]) </ins> if float64(memAllocatable)-(memFraction*float64(memAllocatable)) < float64(2*crioMinMemLimit) { return true }

var cpuFractionMap = make(map[string]float64) var memFractionMap = make(map[string]float64) <ins> // For each node, stores its pods info nodeNameToPodList := podListForEachNode(cs) </ins> for _, node := range nodes { <del> cpuFraction, memFraction, _, _ := computeCPUMemFraction(cs, node, requestedResource) </del> <ins> cpuFraction, memFraction, _, _ := computeCPUMemFraction(node, requestedResource, nodeNameToPodList[node.Name]) </ins> cpuFractionMap[node.Name] = cpuFraction memFractionMap[node.Name] = memFraction if cpuFraction > maxCPUFraction {

} } <ins> nodeNameToPodList = podListForEachNode(cs) </ins> for _, node := range nodes { ginkgo.By("Compute Cpu, Mem Fraction after create balanced pods.") <del> computeCPUMemFraction(cs, node, requestedResource) </del> <ins> computeCPUMemFraction(node, requestedResource, nodeNameToPodList[node.Name]) </ins> } return cleanUp, nil } <del> func computeCPUMemFraction(cs clientset.Interface, node v1.Node, resource *v1.ResourceRequirements) (float64, float64, int64, int64) { framework.Logf("ComputeCPUMemFraction for node: %v", node.Name) totalRequestedCPUResource := resource.Requests.Cpu().MilliValue() totalRequestedMemResource := resource.Requests.Memory().Value() allpods, err := cs.CoreV1().Pods(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) </del> <ins> func podListForEachNode(cs clientset.Interface) map[string][]*v1.Pod { nodeNameToPodList := make(map[string][]*v1.Pod) allPods, err := cs.CoreV1().Pods(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) </ins> if err != nil { framework.Failf("Expect error of invalid, got : %v", err) } <del> for _, pod := range allpods.Items { if pod.Spec.NodeName == node.Name { framework.Logf("Pod for on the node: %v, Cpu: %v, Mem: %v", pod.Name, getNonZeroRequests(&pod).MilliCPU, getNonZeroRequests(&pod).Memory) // Ignore best effort pods while computing fractions as they won't be taken in account by scheduler. if v1qos.GetPodQOS(&pod) == v1.PodQOSBestEffort { continue } totalRequestedCPUResource += getNonZeroRequests(&pod).MilliCPU totalRequestedMemResource += getNonZeroRequests(&pod).Memory </del> <ins> for _, pod := range allPods.Items { nodeName := pod.Spec.NodeName nodeNameToPodList[nodeName] = append(nodeNameToPodList[nodeName], &pod) } return nodeNameToPodList } func computeCPUMemFraction(node v1.Node, resource *v1.ResourceRequirements, pods []*v1.Pod) (float64, float64, int64, int64) { framework.Logf("ComputeCPUMemFraction for node: %v", node.Name) totalRequestedCPUResource := resource.Requests.Cpu().MilliValue() totalRequestedMemResource := resource.Requests.Memory().Value() for _, pod := range pods { framework.Logf("Pod for on the node: %v, Cpu: %v, Mem: %v", pod.Name, getNonZeroRequests(pod).MilliCPU, getNonZeroRequests(pod).Memory) // Ignore best effort pods while computing fractions as they won't be taken in account by scheduler. if v1qos.GetPodQOS(pod) == v1.PodQOSBestEffort { continue </ins> } <ins> totalRequestedCPUResource += getNonZeroRequests(pod).MilliCPU totalRequestedMemResource += getNonZeroRequests(pod).Memory </ins> } cpuAllocatable, found := node.Status.Allocatable[v1.ResourceCPU] framework.ExpectEqual(found, true) cpuAllocatableMil := cpuAllocatable.MilliValue()