Change:
framework.ExpectNoError(err) }) <del> ginkgo.It("should be able to change the type and ports of a service [Slow]", func() { // requires cloud load-balancer support e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws") loadBalancerSupportsUDP := !framework.ProviderIs("aws") loadBalancerLagTimeout := e2eservice.LoadBalancerLagTimeoutDefault if framework.ProviderIs("aws") { loadBalancerLagTimeout = e2eservice.LoadBalancerLagTimeoutAWS } loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs) // This test is more monolithic than we'd like because LB turnup can be // very slow, so we lumped all the tests into one LB lifecycle. serviceName := "mutability-test" ns1 := f.Namespace.Name // LB1 in ns1 on TCP framework.Logf("namespace for TCP test: %s", ns1) ginkgo.By("creating a second namespace") namespacePtr, err := f.CreateNamespace("services", nil) framework.ExpectNoError(err, "failed to create namespace") ns2 := namespacePtr.Name // LB2 in ns2 on UDP framework.Logf("namespace for UDP test: %s", ns2) nodeIP, err := e2enode.PickIP(cs) // for later framework.ExpectNoError(err) // Test TCP and UDP Services.  Services with the same name in different // namespaces should get different node ports and load balancers. ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1) tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName) tcpService, err := tcpJig.CreateTCPService(nil) framework.ExpectNoError(err) ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in namespace " + ns2) udpJig := e2eservice.NewTestJig(cs, ns2, serviceName) udpService, err := udpJig.CreateUDPService(nil) framework.ExpectNoError(err) ginkgo.By("verifying that TCP and UDP use the same port") if tcpService.Spec.Ports[0].Port != udpService.Spec.Ports[0].Port { framework.Failf("expected to use the same port for TCP and UDP") } svcPort := int(tcpService.Spec.Ports[0].Port) framework.Logf("service port (TCP and UDP): %d", svcPort) ginkgo.By("creating a pod to be part of the TCP service " + serviceName) _, err = tcpJig.Run(nil) framework.ExpectNoError(err) ginkgo.By("creating a pod to be part of the UDP service " + serviceName) _, err = udpJig.Run(nil) framework.ExpectNoError(err) // Change the services to NodePort. ginkgo.By("changing the TCP service to type=NodePort") tcpService, err = tcpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeNodePort }) framework.ExpectNoError(err) tcpNodePort := int(tcpService.Spec.Ports[0].NodePort) framework.Logf("TCP node port: %d", tcpNodePort) ginkgo.By("changing the UDP service to type=NodePort") udpService, err = udpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeNodePort }) framework.ExpectNoError(err) udpNodePort := int(udpService.Spec.Ports[0].NodePort) framework.Logf("UDP node port: %d", udpNodePort) ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) // Change the services to LoadBalancer. // Here we test that LoadBalancers can receive static IP addresses.  This isn't // necessary, but is an additional feature this monolithic test checks. requestedIP := "" staticIPName := "" if framework.ProviderIs("gce", "gke") { ginkgo.By("creating a static load balancer IP") staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunID) gceCloud, err := gce.GetGCECloud() framework.ExpectNoError(err, "failed to get GCE cloud provider") err = gceCloud.ReserveRegionAddress(&compute.Address{Name: staticIPName}, gceCloud.Region()) defer func() { if staticIPName != "" { // Release GCE static IP - this is not kube-managed and will not be automatically released. if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil { framework.Logf("failed to release static IP %s: %v", staticIPName, err) } } }() framework.ExpectNoError(err, "failed to create region address: %s", staticIPName) reservedAddr, err := gceCloud.GetRegionAddress(staticIPName, gceCloud.Region()) framework.ExpectNoError(err, "failed to get region address: %s", staticIPName) requestedIP = reservedAddr.Address framework.Logf("Allocated static load balancer IP: %s", requestedIP) } ginkgo.By("changing the TCP service to type=LoadBalancer") tcpService, err = tcpJig.UpdateService(func(s *v1.Service) { s.Spec.LoadBalancerIP = requestedIP // will be "" if not applicable s.Spec.Type = v1.ServiceTypeLoadBalancer }) framework.ExpectNoError(err) if loadBalancerSupportsUDP { ginkgo.By("changing the UDP service to type=LoadBalancer") udpService, err = udpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeLoadBalancer }) framework.ExpectNoError(err) } serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(tcpService)) if loadBalancerSupportsUDP { serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(udpService)) } ginkgo.By("waiting for the TCP service to have a load balancer") // Wait for the load balancer to be created asynchronously tcpService, err = tcpJig.WaitForLoadBalancer(loadBalancerCreateTimeout) framework.ExpectNoError(err) if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort { framework.Failf("TCP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", tcpNodePort, tcpService.Spec.Ports[0].NodePort) } if requestedIP != "" && e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP { framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) framework.Logf("TCP load balancer: %s", tcpIngressIP) if framework.ProviderIs("gce", "gke") { // Do this as early as possible, which overrides the `defer` above. // This is mostly out of fear of leaking the IP in a timeout case // (as of this writing we're not 100% sure where the leaks are // coming from, so this is first-aid rather than surgery). ginkgo.By("demoting the static IP to ephemeral") if staticIPName != "" { gceCloud, err := gce.GetGCECloud() framework.ExpectNoError(err, "failed to get GCE cloud provider") // Deleting it after it is attached "demotes" it to an // ephemeral IP, which can be auto-released. if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil { framework.Failf("failed to release static IP %s: %v", staticIPName, err) } staticIPName = "" } } var udpIngressIP string if loadBalancerSupportsUDP { ginkgo.By("waiting for the UDP service to have a load balancer") // 2nd one should be faster since they ran in parallel. udpService, err = udpJig.WaitForLoadBalancer(loadBalancerCreateTimeout) framework.ExpectNoError(err) if int(udpService.Spec.Ports[0].NodePort) != udpNodePort { framework.Failf("UDP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", udpNodePort, udpService.Spec.Ports[0].NodePort) } udpIngressIP = e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) framework.Logf("UDP load balancer: %s", udpIngressIP) ginkgo.By("verifying that TCP and UDP use different load balancers") if tcpIngressIP == udpIngressIP { framework.Failf("Load balancers are not different: %s", e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } } ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) if loadBalancerSupportsUDP { ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) } // Change the services' node ports. ginkgo.By("changing the TCP service's NodePort") tcpService, err = tcpJig.ChangeServiceNodePort(tcpNodePort) framework.ExpectNoError(err) tcpNodePortOld := tcpNodePort tcpNodePort = int(tcpService.Spec.Ports[0].NodePort) if tcpNodePort == tcpNodePortOld { framework.Failf("TCP Spec.Ports[0].NodePort (%d) did not change", tcpNodePort) } if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP { framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("TCP node port: %d", tcpNodePort) ginkgo.By("changing the UDP service's NodePort") udpService, err = udpJig.ChangeServiceNodePort(udpNodePort) framework.ExpectNoError(err) udpNodePortOld := udpNodePort udpNodePort = int(udpService.Spec.Ports[0].NodePort) if udpNodePort == udpNodePortOld { framework.Failf("UDP Spec.Ports[0].NodePort (%d) did not change", udpNodePort) } if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP { framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("UDP node port: %d", udpNodePort) ginkgo.By("hitting the TCP service's new NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's new NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the old TCP NodePort is closed") testNotReachableHTTP(nodeIP, tcpNodePortOld, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the old UDP NodePort is closed") testNotReachableUDP(nodeIP, udpNodePortOld, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) if loadBalancerSupportsUDP { ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) } // Change the services' main ports. ginkgo.By("changing the TCP service's port") tcpService, err = tcpJig.UpdateService(func(s *v1.Service) { s.Spec.Ports[0].Port++ }) framework.ExpectNoError(err) svcPortOld := svcPort svcPort = int(tcpService.Spec.Ports[0].Port) if svcPort == svcPortOld { framework.Failf("TCP Spec.Ports[0].Port (%d) did not change", svcPort) } if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort { framework.Failf("TCP Spec.Ports[0].NodePort (%d) changed", tcpService.Spec.Ports[0].NodePort) } if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP { framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])) } ginkgo.By("changing the UDP service's port") udpService, err = udpJig.UpdateService(func(s *v1.Service) { s.Spec.Ports[0].Port++ }) framework.ExpectNoError(err) if int(udpService.Spec.Ports[0].Port) != svcPort { framework.Failf("UDP Spec.Ports[0].Port (%d) did not change", udpService.Spec.Ports[0].Port) } if int(udpService.Spec.Ports[0].NodePort) != udpNodePort { framework.Failf("UDP Spec.Ports[0].NodePort (%d) changed", udpService.Spec.Ports[0].NodePort) } if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP { framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])) } framework.Logf("service port (TCP and UDP): %d", svcPort) ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) if loadBalancerSupportsUDP { ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) } ginkgo.By("Scaling the pods to 0") err = tcpJig.Scale(0) framework.ExpectNoError(err) err = udpJig.Scale(0) framework.ExpectNoError(err) ginkgo.By("looking for ICMP REJECT on the TCP service's NodePort") testRejectedHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("looking for ICMP REJECT on the UDP service's NodePort") testRejectedUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("looking for ICMP REJECT on the TCP service's LoadBalancer") testRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) if loadBalancerSupportsUDP { ginkgo.By("looking for ICMP REJECT on the UDP service's LoadBalancer") testRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) } ginkgo.By("Scaling the pods to 1") err = tcpJig.Scale(1) framework.ExpectNoError(err) err = udpJig.Scale(1) framework.ExpectNoError(err) ginkgo.By("hitting the TCP service's NodePort") e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the UDP service's NodePort") testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("hitting the TCP service's LoadBalancer") e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout) if loadBalancerSupportsUDP { ginkgo.By("hitting the UDP service's LoadBalancer") testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout) } // Change the services back to ClusterIP. ginkgo.By("changing TCP service back to type=ClusterIP") tcpReadback, err := tcpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeClusterIP }) framework.ExpectNoError(err) if tcpReadback.Spec.Ports[0].NodePort != 0 { framework.Fail("TCP Spec.Ports[0].NodePort was not cleared") } // Wait for the load balancer to be destroyed asynchronously _, err = tcpJig.WaitForLoadBalancerDestroy(tcpIngressIP, svcPort, loadBalancerCreateTimeout) framework.ExpectNoError(err) ginkgo.By("changing UDP service back to type=ClusterIP") udpReadback, err := udpJig.UpdateService(func(s *v1.Service) { s.Spec.Type = v1.ServiceTypeClusterIP }) framework.ExpectNoError(err) if udpReadback.Spec.Ports[0].NodePort != 0 { framework.Fail("UDP Spec.Ports[0].NodePort was not cleared") } if loadBalancerSupportsUDP { // Wait for the load balancer to be destroyed asynchronously _, err = udpJig.WaitForLoadBalancerDestroy(udpIngressIP, svcPort, loadBalancerCreateTimeout) framework.ExpectNoError(err) } ginkgo.By("checking the TCP NodePort is closed") testNotReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the UDP NodePort is closed") testNotReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout) ginkgo.By("checking the TCP LoadBalancer is closed") testNotReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout) if loadBalancerSupportsUDP { ginkgo.By("checking the UDP LoadBalancer is closed") testNotReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout) } }) </del> /* Testname: Service, update NodePort, same port different protocol Description: Create a service to accept TCP requests. By default, created service MUST be of type ClusterIP and an ClusterIP MUST be assigned to the service.

Discussion:
Which jobs are flaking: gce-master-scale-correctness Which test(s) are flaking: Services should be able to change the type and ports of a service [Slow] Testgrid link: https://k8s- Reason for failure: Test timeouts in ~half of the runs since it was enabled here: /sig scalability /sig network
/cc /cc
The context is - the problems are strictly related to scale - 5k-node jobs are timing out. I very briefly skimmed through the test and didn't see anything obviously suspicious (apart from the fact there are gazillions long timeouts there, which alltogether sum up long hours). That said - it seems to me that there are effectively two separate tests inside it: one for TCP one for UDP They don't seem to share anything with each other. - would it make sense to split this test two effectively two separate tests: one for TCP and one for UDP. (I guess it would also allow us to share some code there).
[And potentially it could help with debugging the problem if it is strictly related to only one of those cases...]
I wanted to focus on this past week but I didn´t have time, this is my list of more expensive times and approx average duration, all of the sig-network ones use LoadBalancers. I wanted to time the test to understand if there are places where can save time ... I don´t have much context on those tests, maybe Tim can help us more, but this comment makes me think that the use of both protocols is intended
/triage accepted
/triage accepted It's been a very long time since I touched this, and I recall holding my nose on it in the first place. Splitting TCP from UDP seems reasonable if we can run them in parallel. we'll want to share the flow, and parameterize some shared funcs. I don't know that this will prevent test timeouts, though.
We could use a volunteer to go poke at this, and see how hard it will really be.
You mean "forcing them to run in parallel"? Or "allowing them to run in parallel"? The later doesn't require any additiona work. The former I think mean that it will still have to be a single test (just the code will be organized better) and that won't really help with debugging (nor preventing timeouts).

this was discussed yesterday in sig-net meeting and my understanding was to split current test in 2 different so the test can run in parallel, if you want to do so and one protocol doesn´t have to wait for the other. I think that is what you meant here
As long as we CAN parallelize. Creating LBs can be sloooooow. :P Antonio Ojea <:
Reopening - I don't think just splitting will address the problem. It may help us with debugging though...
With the split we moved from 60 mins to run the test, to run 2 tests in [30-40] mins each https://k8s- Seems a win :smile:
Awesome - that's something I was silently counting for TBH :) [I doubt it will fix flakiness, but maybe it will make it easier to debug...]
Actually - I think we're getting close thanks to the two changes ( + this one). Here is the run: This particular one filed on UDP (not sure if it's just UDP-related, or just manifested with UDP now). And we have a clear failure now: In my PR I decreased the timeout from 2h to 45m, as it seemed to me that in the past it should be enough. But it failed on 45m timeout here. So I think the next question now is: is it that slow - that we need more than 45m or something isn't working as expected (either in the k8s itself or in the test) But I think we have a clear single place to look at now.
Actually - the full stacktrace: So it's this line that failed: What is interesting is that something was changing during those 45 minutes:
Note that it's 4th time we actually wait for LB in this test. The failure happens after the pod were scaled down to 0 and then to 1. So that makes me think, that's it's actually not LB-related, but rather something internally in k8s, becuase L4 LB doesn't depend on pods - it targets nodes...
If it is always UDP, I start to see ghosts between GCE and UDP ... we only have KIND to compare against, or do we have some other non gce enviroment?

I mean, there are other tests failing in UDP , and show much more failures in GCE than in kind
- this does not look "normal". Can we give some prio to helping track this flake?
hmm interesting. we will investigate.
/assign
- any progress on this?
I haven't had a chance to look into this yet. I will prioritize it this week.
Digging through the logs I believe this was a bug in the test code that was fixed by . In the test, the service port changes around 12:33. However due to the bug, the test continues to use the old service port as seen in the test output: The port should be 81. I believe the following sequence occurs from this point: 1) The reachability test fails in the beginning because the service has changed the port and the load balancer has not been updated yet resulting in the error. 1) Around 12:38, the service controller is syncing the service and notices that the service port has changed and starts to delete the forwarding rule causing the error to change to . 1) Once the forwarding rule is recreated, the error goes back to being because the forwarding rule exists but the wrong port is used. Looking at , the test runs after the one linked have all passed. Is there anything else that needs to be investigated as part of this issue?
OK - the test was quite flaky in the past but indeed it didn't flake for the last 3 weeks (neither this nor TCP one). I'm fine with closing this one - we can open a new one if this happens again. - thanks for debugging. - it seems that the splitting of the test actually made things waay better. Which is great.