Change:
<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: Which test(s) are failing: TaintBasedEvictions Since when has it been failing: probably since it was in this PR? Testgrid link: Reason for failure: Anything else we need to know: /sig scheduling /kind flake /priority important-soon /milestone v1.17
You must be a member of the GitHub team to set the milestone. If you believe you should be able to issue the /milestone command, please contact your and have them propose you as an additional delegate for this responsibility. <detailsIn response to : <!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs -- TaintBasedEvictions probably since it was in this PR? Anything else we need to know: /sig scheduling /kind flake /priority important-soon /milestone v1.17 </details
cc.
Looking into this. We moved it to integration because it was flaking in e2e too
/assign damemi
Not sure yet, but I do see this in the logs of the failing tests: which, if I understand this, would remove all the taints and mark nodes as reachable (which would explain why the test is failing waiting for node taints: ). But I need to figure out why all the nodes are becoming unavailable
is that a full fix for this flake or just partial? I still saw it failed yesterday so just checking... :)
I think I also run into this on 13th Oct: Full logs:
Now gets merged, which might fix the flake. Let's keep an eye on this.
reported it still flaked: Can you continue looking into this?
I'll keep trying, from my previous debugging it seems that in some cases the test cluster is entering full disruption mode and removing all of the taints from the nodes. I know we specifically made changes to the test (using multiple nodes) to ensure it wouldn't become fully disrupted, do you have any idea what could still be causing that?
There is some flag controlling how the dispution thing is managed internally, maybe you can check how a node lifecycle manager is constructed.
The test still flakes after we merge , we still need to investigate the cause ,
/assign
Another flake:
These flakes are consistently (always) timeouts waiting on node taints:
I'm working on this. Will send out a PR soon.
/reopen We see another flake since got merged: However, don't panic :) The original "Entering master disruption mode" cause has been fixed - you won't see that message in the above log. This time it's a smaller issue. I will explain it in another PR.
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen The job is still flaky
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/close But for another test :)
Closing this issue. <detailsIn response to : /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
// This file tests the Taint feature. import ( <ins> "errors" </ins> "fmt" "testing" "time" v1 "k8s.io/api/core/v1" <ins> apierrors "k8s.io/apimachinery/pkg/api/errors" </ins> "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema"

nodeCount := 3 zero := int64(0) gracePeriod := int64(1) <ins> heartbeatInternal := time.Second * 2 </ins> testPod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "testpod1", DeletionGracePeriodSeconds: &zero}, Spec: v1.PodSpec{

for i, test := range tests { t.Run(test.name, func(t *testing.T) { context := initTestMaster(t, "taint-based-evictions", admission) <del> defer cleanupTest(t, context) </del> // Build clientset and informers for controllers. externalClientset := kubernetes.NewForConfigOrDie(&restclient.Config{

podTolerations.SetExternalKubeInformerFactory(externalInformers) context = initTestScheduler(t, context, true, nil) <ins> defer cleanupTest(t, context) </ins> cs := context.clientSet informers := context.informerFactory _, err := cs.CoreV1().Namespaces().Create(context.ns)

Allocatable: nodeRes, Conditions: []v1.NodeCondition{ { <del> Type:   v1.NodeReady, Status: v1.ConditionTrue, </del> <ins> Type:              v1.NodeReady, Status:            v1.ConditionTrue, LastHeartbeatTime: metav1.Now(), </ins> }, }, },

} } <del> // Regularly send heartbeat event to APIServer so that the cluster doesn't enter fullyDisruption mode. // TODO(Huang-Wei): use "NodeDisruptionExclusion" feature to simply the below logic when it's beta. var heartbeatChans []chan struct{} for i := 0; i < nodeCount; i++ { heartbeatChans = append(heartbeatChans, make(chan struct{})) } for i := 0; i < nodeCount; i++ { // Spin up <nodeCount> goroutines to send heartbeat event to APIServer periodically. go func(i int) { for { select { case <-heartbeatChans[i]: return case <-time.Tick(2 * time.Second): nodes[i].Status.Conditions = []v1.NodeCondition{ { Type:              v1.NodeReady, Status:            v1.ConditionTrue, LastHeartbeatTime: metav1.Now(), }, } updateNodeStatus(cs, nodes[i]) } } }(i) } </del> neededNode := nodes[1] if test.pod != nil { test.pod.Name = fmt.Sprintf("testpod-%d", i)

} } <ins> // Regularly send heartbeat event to APIServer so that the cluster doesn't enter fullyDisruption mode. // TODO(Huang-Wei): use "NodeDisruptionExclusion" feature to simply the below logic when it's beta. </ins> for i := 0; i < nodeCount; i++ { <del> // Stop the neededNode's heartbeat goroutine. if neededNode.Name == fmt.Sprintf("node-%d", i) { heartbeatChans[i] <- struct{}{} break </del> <ins> var conditions []v1.NodeCondition // If current node is not <neededNode> if neededNode.Name != nodes[i].Name { conditions = []v1.NodeCondition{ { Type:   v1.NodeReady, Status: v1.ConditionTrue, }, } } else { c, err := nodeReadyStatus(test.nodeConditions) if err != nil { t.Error(err) } // Need to distinguish NodeReady/False and NodeReady/Unknown. // If we try to update the node with condition NotReady/False, i.e. expect a NotReady:NoExecute taint // we need to keep sending the update event to keep it alive, rather than just sending once. if c == v1.ConditionFalse { conditions = test.nodeConditions } else if c == v1.ConditionUnknown { // If it's expected to update the node with condition NotReady/Unknown, // i.e. expect a Unreachable:NoExecute taint, // we need to only send the update event once to simulate the network unreachable scenario. nodeCopy := nodeCopyWithConditions(nodes[i], test.nodeConditions) if err := updateNodeStatus(cs, nodeCopy); err != nil && !apierrors.IsNotFound(err) { t.Errorf("Cannot update node: %v", err) } continue } </ins> } <del> } neededNode.Status.Conditions = test.nodeConditions // Update node condition. err = updateNodeStatus(cs, neededNode) if err != nil { t.Fatalf("Cannot update node: %v", err) </del> <ins> // Keeping sending NodeReady/True or NodeReady/False events. go func(i int) { for { select { case <-context.ctx.Done(): return case <-time.Tick(heartbeatInternal): nodeCopy := nodeCopyWithConditions(nodes[i], conditions) if err := updateNodeStatus(cs, nodeCopy); err != nil && !apierrors.IsNotFound(err) { t.Errorf("Cannot update node: %v", err) } } } }(i) </ins> } if err := waitForNodeTaints(cs, neededNode, test.nodeTaints); err != nil {

} cleanupPods(cs, t, []*v1.Pod{test.pod}) } <del> // Close all heartbeat channels. for i := 0; i < nodeCount; i++ { close(heartbeatChans[i]) } </del> cleanupNodes(cs, t) waitForSchedulerCacheCleanup(context.scheduler, t) })

} return 0, fmt.Errorf("cannot find toleration") } <ins> // nodeReadyStatus returns the status of first condition with type NodeReady. // If none of the condition is of type NodeReady, returns an error. func nodeReadyStatus(conditions []v1.NodeCondition) (v1.ConditionStatus, error) { for _, c := range conditions { if c.Type != v1.NodeReady { continue } // Just return the first condition with type NodeReady return c.Status, nil } return v1.ConditionFalse, errors.New("None of the conditions is of type NodeReady") } func nodeCopyWithConditions(node *v1.Node, conditions []v1.NodeCondition) *v1.Node { copy := node.DeepCopy() copy.ResourceVersion = "0" copy.Status.Conditions = conditions for i := range copy.Status.Conditions { copy.Status.Conditions[i].LastHeartbeatTime = metav1.Now() } return copy } </ins>