Code change:
framework.ExpectNoError(verifyServeHostnameServiceUp(cs, ns, host, podNames2, svc2IP, servicePort)) }) <del> ginkgo.It("should be able to preserve UDP traffic when server pod cycles for a NodePort service", func() { serviceName := "clusterip-test" serverPod1Name := "server-1" serverPod2Name := "server-2" ns := f.Namespace.Name nodeIP, err := e2enode.PickIP(cs) // for later framework.ExpectNoError(err) // Create a NodePort service udpJig := e2eservice.NewTestJig(cs, ns, serviceName) ginkgo.By("creating a UDP service " + serviceName + " with type=NodePort in " + ns) udpService, err := udpJig.CreateUDPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeNodePort svc.Spec.Ports = []v1.ServicePort{ {Port: 80, Name: "http", Protocol: v1.ProtocolUDP, TargetPort: intstr.FromInt(80)}, } }) framework.ExpectNoError(err) // Add a backend pod to the service ginkgo.By("creating a backend pod for the service " + serviceName) serverPod1 := newAgnhostPod(serverPod1Name, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod1.Labels = udpJig.Labels _, err = cs.CoreV1().Pods(ns).Create(context.TODO(), serverPod1, metav1.CreateOptions{}) ginkgo.By(fmt.Sprintf("checking NodePort service %s on node with public IP %s", serviceName, nodeIP)) framework.ExpectNoError(err) framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, serverPod1.Name, f.Namespace.Name, framework.PodStartTimeout)) // Waiting for service to expose endpoint. err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{serverPod1Name: {80}}) framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) // Check that the pod reveives the traffic ginkgo.By("Sending UDP traffic to NodePort service " + serviceName + " on node with publicIP " + nodeIP) errorChannel := make(chan error) signal := make(chan struct{}, 1) go continuousEcho(nodeIP, int(udpService.Spec.Ports[0].NodePort), 3*time.Second, 20, signal, errorChannel) // Create a second pod ginkgo.By("creating a second pod for the service " + serviceName) serverPod2 := newAgnhostPod(serverPod2Name, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod2.Labels = udpJig.Labels _, err = cs.CoreV1().Pods(ns).Create(context.TODO(), serverPod2, metav1.CreateOptions{}) framework.ExpectNoError(err) framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, serverPod2.Name, f.Namespace.Name, framework.PodStartTimeout)) // and delete the first pod framework.Logf("Cleaning up %s pod", serverPod1Name) err = cs.CoreV1().Pods(ns).Delete(context.TODO(), serverPod1Name, metav1.DeleteOptions{}) framework.ExpectNoError(err, "failed to delete pod: %s on node", serverPod1Name) // Check that the second pod keeps receiving traffic ginkgo.By("Sending UDP traffic to NodePort service " + serviceName + " on node with publicIP " + nodeIP) signal <- struct{}{} // Check that there are no errors err = <-errorChannel framework.ExpectNoError(err, "pod communication failed") }) </del> /* Release : v1.16 Testname: Service, NodePort Service

Related discussion:
text: 3) Would simulating this in kind by overloading coredns be a possibility as a way to make a more reproducible bug model?
regarding 1) kubeadm does not ship nodeLocalDNSCache I don't think GCE kube-up does by default either
ok, so now im curious about (2) - i.e. what if we just bumped coredns replica count?
interesting, :thinking: why do you think is a DNS thing? the netcat command that polls the services use the IP addresses not the hostname. I think that it has to be related to some iptables/conntrack race or similar,at least in KIND jobs we know that we have an iptables problem that affects network tests, that has to be fixed before this because it affect this test You can instrument the test locally with this and dump the state of iptables and conntrack on failure if you are able to reproduce it
i figured DNS bc i believe its known that conntrack and DNS at scale can get very messy, right ?
one other curious thing about this test is that we only create 2 pods in the recycling event for the svc. that likely means that any real error is hidden by chance..... If we created a 3rd pod, then we might potentially double the failure rate thus making the flakyness go away and expose the real underlying issue .
There's an invocation of dumpConntrack(cs) here [sig-network] Conntrack should be able to preserve UDP traffic when server pod cycles for a ClusterIP service
is updating the logging to run in an infinite loop ..
This is improving in KIND with the new changes we rolled out yesterday, let's see how it evolves this week
- which changes ? Are you saying this is no longer reproducible on Kind?
there was a lot of noise in the CI signal, especially in KIND jobs due to the CNI portmap bug we finally solved yesterday now we should have a better signal and we can go to the bottom because probably the failure will be "legit" and not a consequence, so this comment should apply
Hi ! I ran this a few times on kind and it passed, but I see , so I'm going to try the loop approach you suggested. Here's what I'm using to test:
quick note, <- potential motivation to possibly have runnable on some subset of tests in upstream eventually, i.e.