Code change:
} } <ins> // TestApplyNoExecuteTaintsToNodesEnqueueTwice ensures we taint every node with NoExecute even if enqueued twice func TestApplyNoExecuteTaintsToNodesEnqueueTwice(t *testing.T) { fakeNow := metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC) evictionTimeout := 10 * time.Minute fakeNodeHandler := &testutil.FakeNodeHandler{ Existing: []*v1.Node{ // Unreachable Taint with effect 'NoExecute' should be applied to this node. { ObjectMeta: metav1.ObjectMeta{ Name:              "node0", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), Labels: map[string]string{ v1.LabelTopologyRegion:          "region1", v1.LabelTopologyZone:            "zone1", v1.LabelFailureDomainBetaRegion: "region1", v1.LabelFailureDomainBetaZone:   "zone1", }, }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionUnknown, LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, // Because of the logic that prevents NC from evicting anything when all Nodes are NotReady // we need second healthy node in tests. { ObjectMeta: metav1.ObjectMeta{ Name:              "node1", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), Labels: map[string]string{ v1.LabelTopologyRegion:          "region1", v1.LabelTopologyZone:            "zone1", v1.LabelFailureDomainBetaRegion: "region1", v1.LabelFailureDomainBetaZone:   "zone1", }, }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionTrue, LastHeartbeatTime:  metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, // NotReady Taint with NoExecute effect should be applied to this node. { ObjectMeta: metav1.ObjectMeta{ Name:              "node2", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), Labels: map[string]string{ v1.LabelTopologyRegion:          "region1", v1.LabelTopologyZone:            "zone1", v1.LabelFailureDomainBetaRegion: "region1", v1.LabelFailureDomainBetaZone:   "zone1", }, }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionFalse, LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Clientset: fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testutil.NewPod("pod0", "node0")}}), } healthyNodeNewStatus := v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionTrue, LastHeartbeatTime:  metav1.Date(2017, 1, 1, 12, 10, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC), }, }, } nodeController, _ := newNodeLifecycleControllerFromClient( fakeNodeHandler, evictionTimeout, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealthyThreshold, testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, true) nodeController.now = func() metav1.Time { return fakeNow } nodeController.recorder = testutil.NewFakeRecorder() nodeController.getPodsAssignedToNode = fakeGetPodsAssignedToNode(fakeNodeHandler.Clientset) if err := nodeController.syncNodeStore(fakeNodeHandler); err != nil { t.Errorf("unexpected error: %v", err) } // 1. monitor node health twice, add untainted node once if err := nodeController.monitorNodeHealth(); err != nil { t.Errorf("unexpected error: %v", err) } if err := nodeController.monitorNodeHealth(); err != nil { t.Errorf("unexpected error: %v", err) } // 2. mark node0 healthy node0, err := fakeNodeHandler.Get(context.TODO(), "node0", metav1.GetOptions{}) if err != nil { t.Errorf("Can't get current node0...") return } node0.Status = healthyNodeNewStatus _, err = fakeNodeHandler.UpdateStatus(context.TODO(), node0, metav1.UpdateOptions{}) if err != nil { t.Errorf(err.Error()) return } // add other notReady nodes fakeNodeHandler.Existing = append(fakeNodeHandler.Existing, []*v1.Node{ // Unreachable Taint with effect 'NoExecute' should be applied to this node. { ObjectMeta: metav1.ObjectMeta{ Name:              "node3", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), Labels: map[string]string{ v1.LabelTopologyRegion:          "region1", v1.LabelTopologyZone:            "zone1", v1.LabelFailureDomainBetaRegion: "region1", v1.LabelFailureDomainBetaZone:   "zone1", }, }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionUnknown, LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, // Because of the logic that prevents NC from evicting anything when all Nodes are NotReady // we need second healthy node in tests. { ObjectMeta: metav1.ObjectMeta{ Name:              "node4", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), Labels: map[string]string{ v1.LabelTopologyRegion:          "region1", v1.LabelTopologyZone:            "zone1", v1.LabelFailureDomainBetaRegion: "region1", v1.LabelFailureDomainBetaZone:   "zone1", }, }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionTrue, LastHeartbeatTime:  metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, // NotReady Taint with NoExecute effect should be applied to this node. { ObjectMeta: metav1.ObjectMeta{ Name:              "node5", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), Labels: map[string]string{ v1.LabelTopologyRegion:          "region1", v1.LabelTopologyZone:            "zone1", v1.LabelFailureDomainBetaRegion: "region1", v1.LabelFailureDomainBetaZone:   "zone1", }, }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionFalse, LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }...) if err := nodeController.syncNodeStore(fakeNodeHandler); err != nil { t.Errorf("unexpected error: %v", err) } // 3. start monitor node health again, add untainted node twice, construct UniqueQueue with duplicated node cache if err := nodeController.monitorNodeHealth(); err != nil { t.Errorf("unexpected error: %v", err) } // 4. do NoExecute taint pass // when processing with node0, condition.Status is NodeReady, and return true with default case // then remove the set value and queue value both, the taint job never stuck nodeController.doNoExecuteTaintingPass() // 5. get node3 and node5, see if it has ready got NoExecute taint node3, err := fakeNodeHandler.Get(context.TODO(), "node3", metav1.GetOptions{}) if err != nil { t.Errorf("Can't get current node3...") return } if !taintutils.TaintExists(node3.Spec.Taints, UnreachableTaintTemplate) || len(node3.Spec.Taints) == 0 { t.Errorf("Not found taint %v in %v, which should be present in %s", UnreachableTaintTemplate, node3.Spec.Taints, node3.Name) } node5, err := fakeNodeHandler.Get(context.TODO(), "node5", metav1.GetOptions{}) if err != nil { t.Errorf("Can't get current node5...") return } if !taintutils.TaintExists(node5.Spec.Taints, NotReadyTaintTemplate) || len(node5.Spec.Taints) == 0 { t.Errorf("Not found taint %v in %v, which should be present in %s", NotReadyTaintTemplate, node5.Spec.Taints, node5.Name) } } </ins> func TestSwapUnreachableNotReadyTaints(t *testing.T) { fakeNow := metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC) evictionTimeout := 10 * time.Minute

Related discussion:
text: ): [root ~]# uname -a Linux liabio 4.18.0-147.5.1.el81.x8664 SMP Wed Feb 5 02:00:39 UTC 2020 x8664 x8664 x8664 GNU/Linux Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig cluster-lifecycle /wg cluster-lifecycle
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /sig sig-cluster-lifecycle /wg sig-cluster-lifecycle /committee sig-cluster-lifecycle Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/remove-sig cluster-lifecycle /sig node
Those labels are not set on the issue: <detailsIn response to : /remove-sig cluster-lifecycle /sig node </details
is NoExecute the expected taint for this condition?
- yes it is when running the taint-manager: evictions are usually safe eviction as we don't force delete pods
ref
I see the same, a node which has node condition set to Unknown, does not get an Unreachable taint with NoExecute effect. However I only see this on 1.19, works fine till 1.18.10 Till 1.18.10, such a node gets two Unreachable taints, one with effect NoExecute and one with effect NoSchedule. But on 1.19.3 it only gets the NoSchedule taint. I see that this is the only commit made in that area between 1.18 and 1.19: , could be related?
It totally breaks production readiness of all recent releases, yet doesn't seem to be regarded as a top priority issue. WTF?