Change:
(root)/Kubernetes%%20suite/Probingcontainerwithreadinessprobeshouldnotbereadybeforeinitialdelayandneverrestart/
Saw again in : Stacktrace pod never became ready Expected error: <errors.errorString | 0xc20802df90: { s: "timed out waiting for the condition", } timed out waiting for the condition not to have occurred
Flake again via Observation: test output is pretty crappy as far as pointing to what the actual issue is and what expected behavior is. But appears from test output we're waiting < 1 second? 21:55:05 [It] with readiness probe should not be ready before initial delay and never restart 21:55:05 21:55:05 Sep 16 21:52:05.058: INFO: pod is not yet ready; pod has phase "Pending". 21:55:05 Sep 16 21:52:10.058: INFO: pod is not yet ready; pod has phase "Pending". 21:55:05 Sep 16 21:52:15.058: INFO: pod is not yet ready; pod has phase "Pending". 21:55:05 Sep 16 21:52:20.063: INFO: pod is not yet ready; pod has phase "Pending". 21:55:05 Sep 16 21:52:25.061: INFO: pod is not yet ready; pod has phase "Pending". 21:55:05 Sep 16 21:52:30.058: INFO: pod is not yet ready; pod has phase "Pending". 21:55:05 Sep 16 21:52:35.058: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:52:40.058: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:52:45.076: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:52:50.401: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:52:55.086: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:53:00.067: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:53:05.057: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:53:10.058: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:53:15.069: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:53:20.065: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:53:25.059: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:53:30.059: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 Sep 16 21:53:30.061: INFO: pod is not yet ready; pod has phase "Running". 21:55:05 [AfterEach] Probing container 21:55:05 21:55:05 STEP: Collecting events from namespace "e2e-tests-container-probe-hpzx5". 21:55:05 Sep 16 21:53:30.067: INFO: event for test-webserver--5cf7-11e5-a8c2-: {scheduler } Scheduled: Successfully assigned test-webserver--5cf7-11e5-a8c2- to pull-e2e-7-minion-3dml 21:55:05 Sep 16 21:53:30.067: INFO: event for test-webserver--5cf7-11e5-a8c2-: {kubelet pull-e2e-7-minion-3dml} Pulled: Container image "" already present on machine 21:55:05 Sep 16 21:53:30.067: INFO: event for test-webserver--5cf7-11e5-a8c2-: {kubelet pull-e2e-7-minion-3dml} Created: Created with docker id 21:55:05 Sep 16 21:53:30.067: INFO: event for test-webserver--5cf7-11e5-a8c2-: {kubelet pull-e2e-7-minion-3dml} Started: Started with docker id 21:55:05 Sep 16 21:53:30.067: INFO: event for test-webserver--5cf7-11e5-a8c2-: {kubelet pull-e2e-7-minion-3dml} Pulling: Pulling image "" 21:55:05 Sep 16 21:53:30.067: INFO: event for test-webserver--5cf7-11e5-a8c2-: {kubelet pull-e2e-7-minion-3dml} Pulled: Successfully pulled image "" 21:55:05 Sep 16 21:53:30.067: INFO: event for test-webserver--5cf7-11e5-a8c2-: {kubelet pull-e2e-7-minion-3dml} Created: Created with docker id 21:55:05 Sep 16 21:53:30.067: INFO: event for test-webserver--5cf7-11e5-a8c2-: {kubelet pull-e2e-7-minion-3dml} Started: Started with docker id 21:55:05 Sep 16 21:53:30.103: INFO: POD NODE PHASE GRACE CONDITIONS 21:55:05 Sep 16 21:53:30.103: INFO: addon-test-v1-kbd3g pull-e2e-7-minion-tvyp Running 30s [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: addon-test-v2-bzjbg pull-e2e-7-minion-fi29 Running [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: addon-test-v2-vhtxj pull-e2e-7-minion-jq30 Running [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: test-webserver--5cf7-11e5-a8c2- pull-e2e-7-minion-3dml Running [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: simple-daemon-set- pull-e2e-7-minion-yjuj Running 30s [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: simple-daemon-set-eoc1w pull-e2e-7-minion-3dml Running 30s [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: simple-daemon-set-lffs4 pull-e2e-7-minion-xtbp Running 30s [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: simple-daemon-set-ovu7y pull-e2e-7-minion-tvyp Running 30s [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: pod--5cf8-11e5-926b- pull-e2e-7-minion-3dml Running [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: pod--5cf7-11e5-94b2- pull-e2e-7-minion-jq30 Running [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: pod--5cf7-11e5-97a8- pull-e2e-7-minion-tvyp Running [{Ready False}] 21:55:05 Sep 16 21:53:30.103: INFO: pod--5cf7-11e5-87b4- pull-e2e-7-minion-jq30 Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: frontend-2sz7d pull-e2e-7-minion-xtbp Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: frontend-bqczs pull-e2e-7-minion-yjuj Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: frontend-uegnl pull-e2e-7-minion-3dml Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: redis-master-26q4e pull-e2e-7-minion-tvyp Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: redis-slave-e9grj pull-e2e-7-minion-yjuj Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: redis-slave-h4fzs pull-e2e-7-minion-xtbp Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: update-demo-kitten-d09lw pull-e2e-7-minion-fi29 Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: update-demo-nautilus-q0t4r pull-e2e-7-minion-xtbp Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: update-demo-nautilus-w0zg7 pull-e2e-7-minion-jq30 Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: nginx pull-e2e-7-minion-tvyp Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: redis-master-3zqa6 pull-e2e-7-minion-jq30 Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: nginx pull-e2e-7-minion-3dml Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: nginx pull-e2e-7-minion-jq30 Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: liveness-http pull-e2e-7-minion-3dml Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: liveness-exec pull-e2e-7-minion-fi29 Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: liveness-http pull-e2e-7-minion-tvyp Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: webserver-5ob6g pull-e2e-7-minion-fi29 Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: svc-latency-rc-mla6w pull-e2e-7-minion-3dml Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: var-expansion--5cf7-11e5-87f1- pull-e2e-7-minion-tvyp Running [{Ready False}] 21:55:05 Sep 16 21:53:30.104: INFO: elasticsearch-logging-v1-95tuy pull-e2e-7-minion-fi29 Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: elasticsearch-logging-v1-i4fc1 pull-e2e-7-minion-tvyp Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: fluentd-elasticsearch-pull-e2e-7-minion-3dml pull-e2e-7-minion-3dml Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: fluentd-elasticsearch-pull-e2e-7-minion-fi29 pull-e2e-7-minion-fi29 Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: fluentd-elasticsearch-pull-e2e-7-minion-jq30 pull-e2e-7-minion-jq30 Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: fluentd-elasticsearch-pull-e2e-7-minion-tvyp pull-e2e-7-minion-tvyp Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: fluentd-elasticsearch-pull-e2e-7-minion-xtbp pull-e2e-7-minion-xtbp Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: fluentd-elasticsearch-pull-e2e-7-minion-yjuj pull-e2e-7-minion-yjuj Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kibana-logging-v1-8cltv pull-e2e-7-minion-fi29 Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kube-dns-v9-pyg06 pull-e2e-7-minion-yjuj Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kube-registry-proxy-pull-e2e-7-minion-3dml pull-e2e-7-minion-3dml Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kube-registry-proxy-pull-e2e-7-minion-fi29 pull-e2e-7-minion-fi29 Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kube-registry-proxy-pull-e2e-7-minion-jq30 pull-e2e-7-minion-jq30 Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kube-registry-proxy-pull-e2e-7-minion-tvyp pull-e2e-7-minion-tvyp Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kube-registry-proxy-pull-e2e-7-minion-xtbp pull-e2e-7-minion-xtbp Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kube-registry-proxy-pull-e2e-7-minion-yjuj pull-e2e-7-minion-yjuj Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kube-registry-v0-7kdv4 pull-e2e-7-minion-jq30 Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: kube-ui-v2-nwemf pull-e2e-7-minion-yjuj Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: monitoring-heapster-v8-uwp6n pull-e2e-7-minion-yjuj Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: monitoring-influx-grafana-v1-zkb36 pull-e2e-7-minion-xtbp Running [{Ready True}] 21:55:05 Sep 16 21:53:30.104: INFO: 21:55:05 Sep 16 21:53:30.104: INFO: Waiting up to 1m0s for all nodes to be ready 21:55:05 Sep 16 21:53:35.113: INFO: Node pull-e2e-7-minion-3dml condition 1/1: type: Ready, status: True, reason: "KubeletReady", message: "kubelet is posting ready status", last transition time: 2015-09-16 21:50:14 -0700 PDT 21:55:05 Sep 16 21:53:35.113: INFO: Successfully found node pull-e2e-7-minion-3dml readiness to be true 21:55:05 Sep 16 21:53:35.113: INFO: Node pull-e2e-7-minion-fi29 condition 1/1: type: Ready, status: True, reason: "KubeletReady", message: "kubelet is posting ready status", last transition time: 2015-09-16 21:50:16 -0700 PDT 21:55:05 Sep 16 21:53:35.113: INFO: Successfully found node pull-e2e-7-minion-fi29 readiness to be true 21:55:05 Sep 16 21:53:35.113: INFO: Node pull-e2e-7-minion-jq30 condition 1/1: type: Ready, status: True, reason: "KubeletReady", message: "kubelet is posting ready status", last transition time: 2015-09-16 21:50:15 -0700 PDT 21:55:05 Sep 16 21:53:35.113: INFO: Successfully found node pull-e2e-7-minion-jq30 readiness to be true 21:55:05 Sep 16 21:53:35.113: INFO: Node pull-e2e-7-minion-tvyp condition 1/1: type: Ready, status: True, reason: "KubeletReady", message: "kubelet is posting ready status", last transition time: 2015-09-16 21:50:15 -0700 PDT 21:55:05 Sep 16 21:53:35.113: INFO: Successfully found node pull-e2e-7-minion-tvyp readiness to be true 21:55:05 Sep 16 21:53:35.113: INFO: Node pull-e2e-7-minion-xtbp condition 1/1: type: Ready, status: True, reason: "KubeletReady", message: "kubelet is posting ready status", last transition time: 2015-09-16 21:50:15 -0700 PDT 21:55:05 Sep 16 21:53:35.113: INFO: Successfully found node pull-e2e-7-minion-xtbp readiness to be true 21:55:05 Sep 16 21:53:35.113: INFO: Node pull-e2e-7-minion-yjuj condition 1/1: type: Ready, status: True, reason: "KubeletReady", message: "kubelet is posting ready status", last transition time: 2015-09-16 21:50:15 -0700 PDT 21:55:05 Sep 16 21:53:35.113: INFO: Successfully found node pull-e2e-7-minion-yjuj readiness to be true 21:55:05 STEP: Destroying namespace "e2e-tests-container-probe-hpzx5" for this suite. 21:55:05 21:55:05 21:55:05 • Failure [195.607 seconds] 21:55:05 Probing container 21:55:05 21:55:05 with readiness probe should not be ready before initial delay and never restart [It] 21:55:05 21:55:05 21:55:05 pod never became ready 21:55:05 Expected error: 21:55:05 <errors.errorString | 0xc208124970: { 21:55:05 s: "timed out waiting for the condition", 21:55:05 } 21:55:05 timed out waiting for the condition 21:55:05 not to have occurred 21:55:05 21:55:05 21:55:05 ------------------------------
this would be a good starter project to learn how the tests and the system work. Let me know if you need a hand.
I will in the mean time send a PR to demote this test to flaky so that it does not produce false regression signals in out continuous integration systems.
Looks like it's still flaky as of an hour ago:
Checked out Jenkin builds: kubernetes-e2e-gce: from build 8586 to build 8683, there is no failure caused by such test. kubernetes-e2e-gke-ci: from build 8487 to build 8594, there is no failure caused by such test. I am closing the issue. Please re-open it if you see new incidents. Thanks!
This showed up again in kubernetes-e2e-gce-parallel build 8702:
New failure:
are you working on this? If not I can take it on.
I am closing this one since it never failed for last 50 runs.
I've seen quite a few flaky failures. E.g., Re-opening this issue.
FYI, this e2e test "never" fails when run in serial with other tests: (root)/Kubernetes%%20suite/ProbingcontainerwithreadinessprobeshouldnotbereadybeforeinitialdelayandneverrestartConformance/history/ Similarly it "never" fails in our normal parallel e2e test run (which is, in theory, identical to the per-PR e2e run, but on a different Jenkins instance): (root)/Kubernetes%%20suite/ProbingcontainerwithreadinessprobeshouldnotbereadybeforeinitialdelayandneverrestartConformance/history/ Most (but not all) of the flakes reported in this bug, as far as I can tell, are on the per-PR Jenkins instance. So that's the best place to focus debugging efforts. FYI
Raising to P0
Log output, for convenience:
This is (sort of) a legitimate failure, not a fault of the test infrastructure. The issue is that we use a in a loop to sync pod status after 10 seconds IF AND ONLY IF no other pod syncs (add, update, delete, etc.) have occurred in that time. When the tests run in parallel and hit just right, other pod changes keep resetting that 10 second timer so the pod sync never occurs. Insidentally, I have a PR in flight which will fix this: Here are the possible solutions (not mutually exclusive): nothing, wait for to fix this the periodic pod sync to use a that isn't constantly reset the timeout and hope for the best what do you think about [2]? Here's the rough timeline of events: From this we can see the test timed out by about 5 seconds. I've aggregated the SyncLoop events .
Thanks for digging! Option 2 seems fine. The fix was actually included in my larger PR, which was sitting in the queue for a long time. Let's fix this first since it's a relatively small change.

Discussion:
rootDirectory  string podWorkers     PodWorkers resyncInterval time.Duration <ins> resyncTicker   *time.Ticker </ins> sourcesReady   SourcesReadyFn // sourcesSeen records the sources seen by kubelet. This set is not thread // safe and should only be access by the main kubelet syncloop goroutine.

// state every sync-frequency seconds. Never returns. func (kl *Kubelet) syncLoop(updates <-chan kubetypes.PodUpdate, handler SyncHandler) { glog.Info("Starting kubelet main sync loop.") <ins> kl.resyncTicker = time.NewTicker(kl.resyncInterval) </ins> var housekeepingTimestamp time.Time for { if !kl.containerRuntimeUp() {

// TODO: Do we want to support this? glog.Errorf("Kubelet does not support snapshot update") } <del> case <-time.After(kl.resyncInterval): </del> <ins> case <-kl.resyncTicker.C: </ins> // Periodically syncs all the pods and performs cleanup tasks. glog.V(4).Infof("SyncLoop (periodic sync)") handler.HandlePodSyncs(kl.podManager.GetPods())

kubelet.backOff = util.NewBackOff(time.Second, time.Minute) kubelet.backOff.Clock = fakeClock kubelet.podKillingCh = make(chan *kubecontainer.Pod, 20) <ins> kubelet.resyncInterval = 10 * time.Second </ins> return &TestKubelet{kubelet, fakeRuntime, mockCadvisor, fakeKubeClient, fakeMirrorClient} }

t.Errorf("Unexpected sync loop time: %s, expected 0", loopTime1) } <ins> // Start sync ticker. kubelet.resyncTicker = time.NewTicker(time.Millisecond) </ins> kubelet.syncLoopIteration(make(chan kubetypes.PodUpdate), kubelet) loopTime2 := kubelet.LatestLoopEntryTime() if loopTime2.IsZero() {

kubelet := testKubelet.kubelet kubelet.lastTimestampRuntimeUp = time.Now() kubelet.networkConfigured = true <del> // The syncLoop waits on time.After(resyncInterval), set it really big so that we don't race for // the channel close kubelet.resyncInterval = time.Second * 30 </del> <ins> // The syncLoop waits on the resyncTicker, so we stop it immediately to avoid a race. kubelet.resyncTicker = time.NewTicker(time.Second) kubelet.resyncTicker.Stop() </ins> ch := make(chan kubetypes.PodUpdate) close(ch)