Change:
cc cc Seeing a failure on our downgrade test. Can someone please look at this? https://k8s- Example failure https://k8s- Seeing timeouts on the downgrade test
Since we're so late in the release process and this is the first time we're seeing this fail, going to mark as non-release-blocking for now.
[MILESTONENOTIFIER] Milestone Labels Complete Issue label settings: sig/scheduling: Issue will be escalated to these SIGs if needed. priority/critical-urgent: Never automatically move out of a release milestone; continually escalate to contributor and SIG through all available channels. kind/bug: Fixes a bug discovered during the current release. <detailsAdditional instructions available <a href=""here</aThe commands available for adding these labels are documented <a href=""here</a</details
It fails in more places than just the downgrade job, but when the downgrade job fails that test, that timeout appears to be the consistent error that it fails with
/assign I looked at the logs. The test creates a few pods to saturate the existing nodes and then creates an additional pod and ensures that this last pod stays pending. The problem we observe is that those saturating pods are often not scheduled, causing a test timeout. Those pods use all the resources on the nodes. So, any small changes to the amount of available node resources or even rounding errors could cause those saturating pods to not schedule. I will test this theory by cutting down the amount of resources that the saturating pods use, but I would like to check if the problem is fixed before submitting the PR. Where can I get the steps these cluster downgrade tests take?
The test setups are in the test-infra repo. E.g.
see my analysis in .

Discussion:
) const maxNumberOfPods int64 = 10 <del> const minPodCPURequest int64 = 500 </del> var localStorageVersion = utilversion.MustParseSemantic("v1.8.0-beta.0")

verifyResult(cs, podsNeededForSaturation, 1, ns) }) <del> // This test verifies we don't allow scheduling of pods in a way that sum of limits of pods is greater than machines capacity. // It assumes that cluster add-on pods stay stable and cannot be run in parallel with any other test that touches Nodes or Pods. </del> <ins> // This test verifies we don't allow scheduling of pods in a way that sum of // limits of pods is greater than machines capacity. // It assumes that cluster add-on pods stay stable and cannot be run in parallel // with any other test that touches Nodes or Pods. </ins> // It is so because we need to have precise control on what's running in the cluster. <ins> // Test scenario: // 1. Find the amount CPU resources on each node. // 2. Create one pod with affinity to each node that uses 70% of the node CPU. // 3. Wait for the pods to be scheduled. // 4. Create another pod with no affinity to any node that need 50% of the largest node CPU. // 5. Make sure this additional pod is not scheduled. </ins> It("validates resource limits of pods that are allowed to run [Conformance]", func() { <ins> framework.WaitForStableCluster(cs, masterNodes) </ins> nodeMaxAllocatable := int64(0) nodeToAllocatableMap := make(map[string]int64) for _, node := range nodeList.Items { <ins> nodeReady := false for _, condition := range node.Status.Conditions { if condition.Type == v1.NodeReady && condition.Status == v1.ConditionTrue { nodeReady = true break } } if !nodeReady { continue } // Apply node label to each node framework.AddOrUpdateLabelOnNode(cs, node.Name, "node", node.Name) framework.ExpectNodeHasLabel(cs, node.Name, "node", node.Name) // Find allocatable amount of CPU. </ins> allocatable, found := node.Status.Allocatable[v1.ResourceCPU] Expect(found).To(Equal(true)) nodeToAllocatableMap[node.Name] = allocatable.MilliValue()

nodeMaxAllocatable = allocatable.MilliValue() } } <del> framework.WaitForStableCluster(cs, masterNodes) </del> <ins> // Clean up added labels after this test. defer func() { for nodeName := range nodeToAllocatableMap { framework.RemoveLabelOffNode(cs, nodeName, "node") } }() </ins> pods, err := cs.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}) framework.ExpectNoError(err)

} } <del> var podsNeededForSaturation int milliCpuPerPod := nodeMaxAllocatable / maxNumberOfPods if milliCpuPerPod < minPodCPURequest { milliCpuPerPod = minPodCPURequest } framework.Logf("Using pod capacity: %vm", milliCpuPerPod) for name, leftAllocatable := range nodeToAllocatableMap { framework.Logf("Node: %v has cpu allocatable: %vm", name, leftAllocatable) podsNeededForSaturation += (int)(leftAllocatable / milliCpuPerPod) } By(fmt.Sprintf("Starting additional %v Pods to fully saturate the cluster CPU and trying to start another one", podsNeededForSaturation)) // As the pods are distributed randomly among nodes, // it can easily happen that all nodes are saturated // and there is no need to create additional pods. // StartPods requires at least one pod to replicate. if podsNeededForSaturation > 0 { framework.ExpectNoError(testutils.StartPods(cs, podsNeededForSaturation, ns, "overcommit", *initPausePod(f, pausePodConfig{ Name:   "", Labels: map[string]string{"name": ""}, Resources: &v1.ResourceRequirements{ Limits: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity(milliCpuPerPod, "DecimalSI"), }, Requests: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity(milliCpuPerPod, "DecimalSI"), </del> <ins> By("Starting Pods to consume most of the cluster CPU.") // Create one pod per node that requires 70% of the node remaining CPU. fillerPods := []*v1.Pod{} for nodeName, cpu := range nodeToAllocatableMap { requestedCPU := cpu * 7 / 10 fillerPods = append(fillerPods, createPausePod(f, pausePodConfig{ Name: "filler-pod-" + nodeName, Resources: &v1.ResourceRequirements{ Limits: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity(requestedCPU, "DecimalSI"), }, Requests: v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity(requestedCPU, "DecimalSI"), }, }, Affinity: &v1.Affinity{ NodeAffinity: &v1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ NodeSelectorTerms: []v1.NodeSelectorTerm{ { MatchExpressions: []v1.NodeSelectorRequirement{ { Key:      "node", Operator: v1.NodeSelectorOpIn, Values:   []string{nodeName}, }, }, }, }, </ins> }, }, <del> }), true, framework.Logf)) </del> <ins> }, })) </ins> } <ins> // Wait for filler pods to schedule. for _, pod := range fillerPods { framework.ExpectNoError(framework.WaitForPodRunningInNamespace(cs, pod)) } By("Creating another pod that requires unavailable amount of CPU.") // Create another pod that requires 50% of the largest node CPU resources. // This pod should remain pending as at least 70% of CPU of other nodes in // the cluster are already consumed. </ins> podName := "additional-pod" conf := pausePodConfig{ Name:   podName, Labels: map[string]string{"name": "additional"}, Resources: &v1.ResourceRequirements{ Limits: v1.ResourceList{ <del> v1.ResourceCPU: *resource.NewMilliQuantity(milliCpuPerPod, "DecimalSI"), </del> <ins> v1.ResourceCPU: *resource.NewMilliQuantity(nodeMaxAllocatable*5/10, "DecimalSI"), </ins> }, }, } WaitForSchedulerAfterAction(f, createPausePodAction(f, conf), podName, false) <del> verifyResult(cs, podsNeededForSaturation, 1, ns) </del> <ins> verifyResult(cs, len(fillerPods), 1, ns) </ins> }) // Test Nodes does not have any label, hence it should be impossible to schedule Pod with