Change:
More detail can be seen in e2e test
cc
, Marek Grabowski wrote:
you are running an older version of the test. can you get the latest and try again ? This test had not failed at all yesterday in our automated test runs yesterday. Closing issue for now.
This failure depends on if the pod is really deleted and kube-proxy (the test container on) catch that the endpoint is deleted in 5s . After sending "delete pod" request for 5s, if the pod was still there and kube-proxy still contains this endpoints(this may happen under pressure), hit test can still detect 5 endpoints. At least we should ensure the pod was deleted. Anyway, I think will be helpful to this test. What do you think?
It doesn't matter if the pod is actually deleted or not. We just care that kube-proxy thinks that it should not be part of the service endpoint. Having said that, it still doesn't hurt to do the check that you are doing. I will assign your PR to myself and review it. In the meantime, can you rebase your PR ?
Thanks for your review. The PR is up to date.
Failed twice today in kubernetes-e2e-gce-parallel
One more today. The error message is not very helpful.
18:28:37 STEP: dialing(http) endpointPodIP:endpointHttpPort from node1 18:28:37 STEP: Dialing from node. command:for i in ' wc -l This is the point where the test fails. Am looking into it. We need to have more tries, we just do seq (1 1), instead of seq (6 6). this is causing flakiness.
Fixed by PR

Discussion:
"k8s.io/kubernetes/pkg/api/latest" "k8s.io/kubernetes/pkg/api/unversioned" client "k8s.io/kubernetes/pkg/client/unversioned" <ins> "k8s.io/kubernetes/pkg/labels" </ins> "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/wait" )

pod := config.endpointPods[0] config.getPodClient().Delete(pod.Name, nil) config.endpointPods = config.endpointPods[1:] <del> time.Sleep(5 * time.Second) // wait for kube-proxy to catch up with the pod being deleted. </del> <ins> // wait for pod being deleted. err := waitForPodToDisappear(config.f.Client, config.f.Namespace.Name, pod.Name, labels.Everything(), time.Second, util.ForeverTestTimeout) if err != nil { Failf("Failed to delete %s pod: %v", pod.Name, err) } // wait for endpoint being removed. err = waitForServiceEndpointsNum(config.f.Client, config.f.Namespace.Name, nodePortServiceName, len(config.endpointPods), time.Second, util.ForeverTestTimeout) if err != nil { Failf("Failed to remove endpoint from service: %s", nodePortServiceName) } // wait for kube-proxy to catch up with the pod being deleted. time.Sleep(5 * time.Second) </ins> } func (config *KubeProxyTestConfig) createPod(pod *api.Pod) *api.Pod {

return p, err } <del> // waitForRCPodToDisappear returns nil if the pod from the given replication controller (described by rcName) no longer exists. // In case of failure or too long waiting time, an error is returned. func waitForRCPodToDisappear(c *client.Client, ns, rcName, podName string) error { label := labels.SelectorFromSet(labels.Set(map[string]string{"name": rcName})) return wait.Poll(20*time.Second, 5*time.Minute, func() (bool, error) { </del> <ins> func waitForPodToDisappear(c *client.Client, ns, podName string, label labels.Selector, interval, timeout time.Duration) error { return wait.Poll(interval, timeout, func() (bool, error) { </ins> Logf("Waiting for pod %s to disappear", podName) pods, err := c.Pods(ns).List(label, fields.Everything()) if err != nil {

}) } <ins> // waitForRCPodToDisappear returns nil if the pod from the given replication controller (described by rcName) no longer exists. // In case of failure or too long waiting time, an error is returned. func waitForRCPodToDisappear(c *client.Client, ns, rcName, podName string) error { label := labels.SelectorFromSet(labels.Set(map[string]string{"name": rcName})) return waitForPodToDisappear(c, ns, podName, label, 20*time.Second, 5*time.Minute) } </ins> // waitForService waits until the service appears (exist == true), or disappears (exist == false) func waitForService(c *client.Client, namespace, name string, exist bool, interval, timeout time.Duration) error { err := wait.Poll(interval, timeout, func() (bool, error) {

return nil } <ins> //waitForServiceEndpointsNum waits until the amount of endpoints that implement service to expectNum. func waitForServiceEndpointsNum(c *client.Client, namespace, serviceName string, expectNum int, interval, timeout time.Duration) error { return wait.Poll(interval, timeout, func() (bool, error) { Logf("Waiting for amount of service:%s endpoints to %d", serviceName, expectNum) list, err := c.Endpoints(namespace).List(labels.Everything()) if err != nil { return false, err } for _, e := range list.Items { if e.Name == serviceName && countEndpointsNum(&e) == expectNum { return true, nil } } return false, nil }) } func countEndpointsNum(e *api.Endpoints) int { num := 0 for _, sub := range e.Subsets { num += len(sub.Addresses) } return num } </ins> // waitForReplicationController waits until the RC appears (exist == true), or disappears (exist == false) func waitForReplicationController(c *client.Client, namespace, name string, exist bool, interval, timeout time.Duration) error { err := wait.Poll(interval, timeout, func() (bool, error) {