Change:
/assign /milestone v1.14
(Edit: Removed comment to ask for help, noticed open PR later... For context/help to measure change post merge: This is currently failing about 8% of the time in )
<img width="1095" alt="Screen Shot 2019-03-16 at 11 47 05 AM" src="https://user-"Think this is done

Discussion:
By("verify NodeStatus report period is longer than lease duration") // NodeStatus is reported from node to master when there is some change or // enough time has passed. So for here, keep checking the time diff <del> // between 2 NodeStatus report, until it is longer than lease duration ( // the same as nodeMonitorGracePeriod). heartbeatTime := getNextReadyConditionHeartbeatTime(f.ClientSet, nodeName, metav1.Time{}) </del> <ins> // between 2 NodeStatus report, until it is longer than lease duration // (the same as nodeMonitorGracePeriod), or it doesn't change for at least leaseDuration lastHeartbeatTime := getReadyConditionHeartbeatTime(f.ClientSet, nodeName) lastObserved := time.Now() </ins> Eventually(func() error { <del> nextHeartbeatTime := getNextReadyConditionHeartbeatTime(f.ClientSet, nodeName, heartbeatTime) if nextHeartbeatTime.Time.After(heartbeatTime.Time.Add(leaseDuration)) { return nil </del> <ins> currentHeartbeatTime := getReadyConditionHeartbeatTime(f.ClientSet, nodeName) currentObserved := time.Now() switch { case currentHeartbeatTime == lastHeartbeatTime: if currentObserved.Sub(lastObserved) > 2*leaseDuration { // heartbeat hasn't changed while watching for at least 2*leaseDuration, success! framework.Logf("node status heartbeat is unchanged for %s, was waiting for at least %s, success!", currentObserved.Sub(lastObserved), 2*leaseDuration) return nil } framework.Logf("node status heartbeat is unchanged for %s, waiting for %s", currentObserved.Sub(lastObserved), 2*leaseDuration) return fmt.Errorf("node status heartbeat is unchanged for %s, waiting for %s", currentObserved.Sub(lastObserved), 2*leaseDuration) case currentHeartbeatTime != lastHeartbeatTime: if currentHeartbeatTime.Sub(lastHeartbeatTime) > leaseDuration { // heartbeat time changed, but the diff was greater than leaseDuration, success! framework.Logf("node status heartbeat changed in %s, was waiting for at least %s, success!", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration) return nil } lastHeartbeatTime = currentHeartbeatTime lastObserved = currentObserved framework.Logf("node status heartbeat changed in %s, waiting for %s", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration) return fmt.Errorf("node status heartbeat changed in %s, waiting for %s", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration) </ins> } <del> heartbeatTime = nextHeartbeatTime return fmt.Errorf("node status report period is shorter than lease duration") // Enter next round immediately. }, 5*time.Minute, time.Nanosecond).Should(BeNil()) </del> <ins> return nil }, 5*time.Minute, time.Second).Should(BeNil()) </ins> By("verify node is still in ready status even though node status report is infrequent") // This check on node status is only meaningful when this e2e test is

}) }) <del> func getNextReadyConditionHeartbeatTime(clientSet clientset.Interface, nodeName string, prevHeartbeatTime metav1.Time) metav1.Time { var newHeartbeatTime metav1.Time Eventually(func() error { node, err := clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) if err != nil { return err } _, readyCondition := testutils.GetNodeCondition(&node.Status, corev1.NodeReady) Expect(readyCondition.Status).To(Equal(corev1.ConditionTrue)) newHeartbeatTime = readyCondition.LastHeartbeatTime if prevHeartbeatTime.Before(&newHeartbeatTime) { return nil } return fmt.Errorf("heartbeat has not changed yet") }, 5*time.Minute, 5*time.Second).Should(BeNil()) return newHeartbeatTime </del> <ins> func getReadyConditionHeartbeatTime(clientSet clientset.Interface, nodeName string) time.Time { node, err := clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) Expect(err).To(BeNil()) _, readyCondition := testutils.GetNodeCondition(&node.Status, corev1.NodeReady) Expect(readyCondition.Status).To(Equal(corev1.ConditionTrue)) return readyCondition.LastHeartbeatTime.Time </ins> } func expectLease(lease *coordv1beta1.Lease, nodeName string) error {