Code change:
framework.ExpectNoError(verifyServeHostnameServiceUp(cs, ns, host, podNames2, svc2IP, servicePort)) }) <del> ginkgo.It("should be able to preserve UDP traffic when server pod cycles for a NodePort service", func() { serviceName := "clusterip-test" serverPod1Name := "server-1" serverPod2Name := "server-2" ns := f.Namespace.Name nodeIP, err := e2enode.PickIP(cs) // for later framework.ExpectNoError(err) // Create a NodePort service udpJig := e2eservice.NewTestJig(cs, ns, serviceName) ginkgo.By("creating a UDP service " + serviceName + " with type=NodePort in " + ns) udpService, err := udpJig.CreateUDPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeNodePort svc.Spec.Ports = []v1.ServicePort{ {Port: 80, Name: "http", Protocol: v1.ProtocolUDP, TargetPort: intstr.FromInt(80)}, } }) framework.ExpectNoError(err) // Add a backend pod to the service ginkgo.By("creating a backend pod for the service " + serviceName) serverPod1 := newAgnhostPod(serverPod1Name, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod1.Labels = udpJig.Labels _, err = cs.CoreV1().Pods(ns).Create(context.TODO(), serverPod1, metav1.CreateOptions{}) ginkgo.By(fmt.Sprintf("checking NodePort service %s on node with public IP %s", serviceName, nodeIP)) framework.ExpectNoError(err) framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, serverPod1.Name, f.Namespace.Name, framework.PodStartTimeout)) // Waiting for service to expose endpoint. err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{serverPod1Name: {80}}) framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) // Check that the pod reveives the traffic ginkgo.By("Sending UDP traffic to NodePort service " + serviceName + " on node with publicIP " + nodeIP) errorChannel := make(chan error) signal := make(chan struct{}, 1) go continuousEcho(nodeIP, int(udpService.Spec.Ports[0].NodePort), 3*time.Second, 20, signal, errorChannel) // Create a second pod ginkgo.By("creating a second pod for the service " + serviceName) serverPod2 := newAgnhostPod(serverPod2Name, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod2.Labels = udpJig.Labels _, err = cs.CoreV1().Pods(ns).Create(context.TODO(), serverPod2, metav1.CreateOptions{}) framework.ExpectNoError(err) framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, serverPod2.Name, f.Namespace.Name, framework.PodStartTimeout)) // and delete the first pod framework.Logf("Cleaning up %s pod", serverPod1Name) err = cs.CoreV1().Pods(ns).Delete(context.TODO(), serverPod1Name, metav1.DeleteOptions{}) framework.ExpectNoError(err, "failed to delete pod: %s on node", serverPod1Name) // Check that the second pod keeps receiving traffic ginkgo.By("Sending UDP traffic to NodePort service " + serviceName + " on node with publicIP " + nodeIP) signal <- struct{}{} // Check that there are no errors err = <-errorChannel framework.ExpectNoError(err, "pod communication failed") }) </del> /* Release : v1.16 Testname: Service, NodePort Service

Related discussion:
text: After my second deflaking PR, we seem to be in the stage where: tests are failing on dozens of different things (probably iptables-issues related) tests are failing only on NodePort tests on "Failed to connecto to backend 1" which it would be great if someone else could look into (kube-proxy seemed to be programmed correctly based on logs)
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry. ‍
This has failed again - Linking to specific part of log where timeout occurred .... we see this
So reviewing the above logs note how in STEP: checking client pod connected to the backend 1 on Node IP 10.138.0.5 28 attempts are made all in the 25th second of the 2 minute at 11am Should this Step back off and wait a few seconds in between tries? So looking at this code we see the following
From reading ... the intention expressed in the test code looks good to me. It is not clear to me why there is a mismatch between the 28 tries in the POD Client Logs and the expressed intention running the logContainsFn(podBackend1) every 5 seconds still digging ....
we are trying to catch one failure in this PR that has additional debugging, it dumps the conntrack table of the nodes on a failure
for those following the issue, we've extra debugging to the test so it dumps the whole conntrack table once it finish. We should look that there is NO stale entries, connections from the pod client to the deleted endpoint) and there is conntrack entries to the new endpoint from the pod client.
- we have runs with the conntrack dump now, e.g. this one: would you be able to take a look into it?
not yet :/ , but I think tomorrow can have some time
cc
For anyone reproducing this issue btw, you need a multinode k8s cluster (it isnt doable on kind i guess bc less likeliehood of network failures when running local) -
Curious about two potential hypotheses to fix this, have we looked into either of these? 1) can we confirm that is turned on/off? 2) is it possible that scaling the number of coredns replicas could help ?