Code change:
I just found a cluster like this cc/
do you know if anyone else is currently working on a fix for this? I've already started looking into a fix, would you be willing to review my PR soon?
I don't know of anyone who is working on it.
(I see your PR, and I'll review it.)

Related discussion:
text: if observedReadyCondition.Status != v1.ConditionUnknown { currentReadyCondition.Status = v1.ConditionUnknown currentReadyCondition.Reason = "NodeStatusUnknown" <del> currentReadyCondition.Message = fmt.Sprintf("Kubelet stopped posting node status.") </del> <ins> currentReadyCondition.Message = "Kubelet stopped posting node status." </ins> // LastProbeTime is the last time we heard from kubelet. currentReadyCondition.LastHeartbeatTime = observedReadyCondition.LastHeartbeatTime currentReadyCondition.LastTransitionTime = nc.now() } } <del> // Like NodeReady condition, NodeOutOfDisk was last set longer ago than gracePeriod, so update // it to Unknown (regardless of its current value) in the master. // TODO(madhusudancs): Refactor this with readyCondition to remove duplicated code. _, oodCondition := v1.GetNodeCondition(&node.Status, v1.NodeOutOfDisk) if oodCondition == nil { glog.V(2).Infof("Out of disk condition of node %v is never updated by kubelet", node.Name) node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{ Type:               v1.NodeOutOfDisk, Status:             v1.ConditionUnknown, Reason:             "NodeStatusNeverUpdated", Message:            fmt.Sprintf("Kubelet never posted node status."), LastHeartbeatTime:  node.CreationTimestamp, LastTransitionTime: nc.now(), }) } else { glog.V(4).Infof("node %v hasn't been updated for %+v. Last out of disk condition is: %+v", node.Name, nc.now().Time.Sub(savedNodeStatus.probeTimestamp.Time), oodCondition) if oodCondition.Status != v1.ConditionUnknown { oodCondition.Status = v1.ConditionUnknown oodCondition.Reason = "NodeStatusUnknown" oodCondition.Message = fmt.Sprintf("Kubelet stopped posting node status.") oodCondition.LastTransitionTime = nc.now() </del> <ins> // remaining node conditions should also be set to Unknown remainingNodeConditionTypes := []v1.NodeConditionType{v1.NodeOutOfDisk, v1.NodeMemoryPressure, v1.NodeDiskPressure} nowTimestamp := nc.now() for _, nodeConditionType := range remainingNodeConditionTypes { _, currentCondition := v1.GetNodeCondition(&node.Status, nodeConditionType) if currentCondition == nil { glog.V(2).Infof("Condition %v of node %v was never updated by kubelet", nodeConditionType, node.Name) node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{ Type:               nodeConditionType, Status:             v1.ConditionUnknown, Reason:             "NodeStatusNeverUpdated", Message:            "Kubelet never posted node status.", LastHeartbeatTime:  node.CreationTimestamp, LastTransitionTime: nowTimestamp, }) } else { glog.V(4).Infof("node %v hasn't been updated for %+v. Last %v is: %+v", node.Name, nc.now().Time.Sub(savedNodeStatus.probeTimestamp.Time), nodeConditionType, currentCondition) if currentCondition.Status != v1.ConditionUnknown { currentCondition.Status = v1.ConditionUnknown currentCondition.Reason = "NodeStatusUnknown" currentCondition.Message = "Kubelet stopped posting node status." currentCondition.LastTransitionTime = nowTimestamp } </ins> } }