Change:
<!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"8+", GitVersion:"v1.8.0-alpha.1.51+-dirty", GitCommit:"", GitTreeState:"dirty", BuildDate:"2017-06-20T10:51:10Z", GoVersion:"go1.8", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"8+", GitVersion:"v1.8.0-alpha.1.51+-dirty", GitCommit:"", GitTreeState:"dirty", BuildDate:"2017-06-20T11:29:14Z", GoVersion:"go1.8", Compiler:"gc", Platform:"linux/amd64"} What happened: After adding label to a node, node controller panics. What you expected to happen: Everything should still work. How to reproduce it (as minimally and precisely as possible): Run and wait for the node controller to panic: Anything else we need to know: I've checked the code and it looks like the node controller expects the label to be present during node registration and it can not be later. Even if the label can not be latter it shouldn't panic.
/sig node
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <br(2) specifying the label manually: <br<brNote: method (1) will trigger a notification to the team. You can find the team list and label list
/assign let me try this :).

Discussion:
<-stopCh } <ins> // addPodEvictorForNewZone checks if new zone appeared, and if so add new evictor. func (nc *NodeController) addPodEvictorForNewZone(node *v1.Node) { zone := utilnode.GetZoneKey(node) if _, found := nc.zoneStates[zone]; !found { nc.zoneStates[zone] = stateInitial if !nc.useTaintBasedEvictions { nc.zonePodEvictor[zone] = NewRateLimitedTimedQueue( flowcontrol.NewTokenBucketRateLimiter(nc.evictionLimiterQPS, evictionRateLimiterBurst)) } else { nc.zoneNotReadyOrUnreachableTainer[zone] = NewRateLimitedTimedQueue( flowcontrol.NewTokenBucketRateLimiter(nc.evictionLimiterQPS, evictionRateLimiterBurst)) } // Init the metric for the new zone. glog.Infof("Initializing eviction metric for zone: %v", zone) EvictionsNumber.WithLabelValues(zone).Add(0) } } </ins> // monitorNodeStatus verifies node status are constantly updated by kubelet, and if not, // post "NodeReady==ConditionUnknown". It also evicts all pods if node is not ready or // not reachable for a long period of time.

if err != nil { return err } <del> added, deleted := nc.checkForNodeAddedDeleted(nodes) </del> <ins> added, deleted, newZoneRepresentatives := nc.classifyNodes(nodes) for i := range newZoneRepresentatives { nc.addPodEvictorForNewZone(newZoneRepresentatives[i]) } </ins> for i := range added { glog.V(1).Infof("NodeController observed a new Node: %#v", added[i].Name) recordNodeEvent(nc.recorder, added[i].Name, string(added[i].UID), v1.EventTypeNormal, "RegisteredNode", fmt.Sprintf("Registered Node %v in NodeController", added[i].Name)) nc.knownNodeSet[added[i].Name] = added[i] <del> // When adding new Nodes we need to check if new zone appeared, and if so add new evictor. zone := utilnode.GetZoneKey(added[i]) if _, found := nc.zoneStates[zone]; !found { nc.zoneStates[zone] = stateInitial if !nc.useTaintBasedEvictions { nc.zonePodEvictor[zone] = NewRateLimitedTimedQueue( flowcontrol.NewTokenBucketRateLimiter(nc.evictionLimiterQPS, evictionRateLimiterBurst)) } else { nc.zoneNotReadyOrUnreachableTainer[zone] = NewRateLimitedTimedQueue( flowcontrol.NewTokenBucketRateLimiter(nc.evictionLimiterQPS, evictionRateLimiterBurst)) } // Init the metric for the new zone. glog.Infof("Initializing eviction metric for zone: %v", zone) EvictionsNumber.WithLabelValues(zone).Add(0) } </del> <ins> nc.addPodEvictorForNewZone(added[i]) </ins> if nc.useTaintBasedEvictions { nc.markNodeAsHealthy(added[i]) } else {

return gracePeriod, observedReadyCondition, currentReadyCondition, err } <del> func (nc *NodeController) checkForNodeAddedDeleted(nodes []*v1.Node) (added, deleted []*v1.Node) { for i := range nodes { if _, has := nc.knownNodeSet[nodes[i].Name]; !has { added = append(added, nodes[i]) </del> <ins> // classifyNodes classifies the allNodes to three categories: //   1. added: the nodes that in 'allNodes', but not in 'knownNodeSet' //   2. deleted: the nodes that in 'knownNodeSet', but not in 'allNodes' //   3. newZoneRepresentatives: the nodes that in both 'knownNodeSet' and 'allNodes', but no zone states func (nc *NodeController) classifyNodes(allNodes []*v1.Node) (added, deleted, newZoneRepresentatives []*v1.Node) { for i := range allNodes { if _, has := nc.knownNodeSet[allNodes[i].Name]; !has { added = append(added, allNodes[i]) } else { // Currently, we only consider new zone as updated. zone := utilnode.GetZoneKey(allNodes[i]) if _, found := nc.zoneStates[zone]; !found { newZoneRepresentatives = append(newZoneRepresentatives, allNodes[i]) } </ins> } } // If there's a difference between lengths of known Nodes and observed nodes // we must have removed some Node. <del> if len(nc.knownNodeSet)+len(added) != len(nodes) { </del> <ins> if len(nc.knownNodeSet)+len(added) != len(allNodes) { </ins> knowSetCopy := map[string]*v1.Node{} for k, v := range nc.knownNodeSet { knowSetCopy[k] = v } <del> for i := range nodes { delete(knowSetCopy, nodes[i].Name) </del> <ins> for i := range allNodes { delete(knowSetCopy, allNodes[i].Name) </ins> } for i := range knowSetCopy { deleted = append(deleted, knowSetCopy[i])

func TestMonitorNodeStatusEvictPods(t *testing.T) { fakeNow := metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC) evictionTimeout := 10 * time.Minute <ins> labels := map[string]string{ kubeletapis.LabelZoneRegion:        "region1", kubeletapis.LabelZoneFailureDomain: "zone1", } </ins> // Because of the logic that prevents NC from evicting anything when all Nodes are NotReady // we need second healthy node in tests. Because of how the tests are written we need to update

expectedEvictPods:   false, description:         "Node created recently, with no status.", }, <ins> // Node created recently without FailureDomain labels which is added back later, with no status (happens only at cluster startup). { fakeNodeHandler: &testutil.FakeNodeHandler{ Existing: []*v1.Node{ { ObjectMeta: metav1.ObjectMeta{ Name:              "node0", CreationTimestamp: fakeNow, }, }, { ObjectMeta: metav1.ObjectMeta{ Name:              "node1", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:               v1.NodeReady, Status:             v1.ConditionTrue, LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Clientset: fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testutil.NewPod("pod0", "node0")}}), }, daemonSets:          nil, timeToPass:          0, newNodeStatus:       v1.NodeStatus{}, secondNodeNewStatus: healthyNodeNewStatus, expectedEvictPods:   false, description:         "Node created recently without FailureDomain labels which is added back later, with no status (happens only at cluster startup).", }, </ins> // Node created long time ago, and kubelet posted NotReady for a short period of time. { fakeNodeHandler: &testutil.FakeNodeHandler{

item.fakeNodeHandler.Existing[0].Status = item.newNodeStatus item.fakeNodeHandler.Existing[1].Status = item.secondNodeNewStatus } <ins> if len(item.fakeNodeHandler.Existing[0].Labels) == 0 && len(item.fakeNodeHandler.Existing[1].Labels) == 0 { item.fakeNodeHandler.Existing[0].Labels = labels item.fakeNodeHandler.Existing[1].Labels = labels } </ins> if err := syncNodeStore(nodeController, item.fakeNodeHandler); err != nil { t.Errorf("unexpected error: %v", err) }