Change:
}) }) <del> var _ = SIGDescribe("ESIPP [Slow]", func() { f := framework.NewDefaultFramework("esipp") var loadBalancerCreateTimeout time.Duration var cs clientset.Interface serviceLBNames := []string{} ginkgo.BeforeEach(func() { // requires cloud load-balancer support - this feature currently supported only on GCE/GKE e2eskipper.SkipUnlessProviderIs("gce", "gke") cs = f.ClientSet loadBalancerCreateTimeout = e2eservice.GetServiceLoadBalancerCreationTimeout(cs) }) ginkgo.AfterEach(func() { if ginkgo.CurrentGinkgoTestDescription().Failed { DescribeSvc(f.Namespace.Name) } for _, lb := range serviceLBNames { framework.Logf("cleaning load balancer resource for %s", lb) e2eservice.CleanupServiceResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone) } //reset serviceLBNames serviceLBNames = []string{} }) ginkgo.It("should work for type=LoadBalancer", func() { namespace := f.Namespace.Name serviceName := "external-local-lb" jig := e2eservice.NewTestJig(cs, namespace, serviceName) svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc)) healthCheckNodePort := int(svc.Spec.HealthCheckNodePort) if healthCheckNodePort == 0 { framework.Failf("Service HealthCheck NodePort was not allocated") } defer func() { err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout) framework.ExpectNoError(err) // Make sure we didn't leak the health check node port. const threshold = 2 nodes, err := getEndpointNodesWithInternalIP(jig) framework.ExpectNoError(err) config := e2enetwork.NewNetworkingTestConfig(f) for _, internalIP := range nodes { err := testHTTPHealthCheckNodePortFromTestContainer( config, internalIP, healthCheckNodePort, e2eservice.KubeProxyLagTimeout, false, threshold) framework.ExpectNoError(err) } err = cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() svcTCPPort := int(svc.Spec.Ports[0].Port) ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) ginkgo.By("reading clientIP using the TCP service's service port via its external VIP") clientIP, err := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip") framework.ExpectNoError(err) framework.Logf("ClientIP detected by target pod using VIP:SvcPort is %s", clientIP) ginkgo.By("checking if Source IP is preserved") if strings.HasPrefix(clientIP, "10.") { framework.Failf("Source IP was NOT preserved") } }) ginkgo.It("should work for type=NodePort", func() { namespace := f.Namespace.Name serviceName := "external-local-nodeport" jig := e2eservice.NewTestJig(cs, namespace, serviceName) svc, err := jig.CreateOnlyLocalNodePortService(true) framework.ExpectNoError(err) defer func() { err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() tcpNodePort := int(svc.Spec.Ports[0].NodePort) endpointsNodeMap, err := getEndpointNodesWithInternalIP(jig) framework.ExpectNoError(err) dialCmd := "clientip" config := e2enetwork.NewNetworkingTestConfig(f) for nodeName, nodeIP := range endpointsNodeMap { ginkgo.By(fmt.Sprintf("reading clientIP using the TCP service's NodePort, on node %v: %v:%v/%v", nodeName, nodeIP, tcpNodePort, dialCmd)) clientIP, err := GetHTTPContentFromTestContainer(config, nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout, dialCmd) framework.ExpectNoError(err) framework.Logf("ClientIP detected by target pod using NodePort is %s, the ip of test container is %s", clientIP, config.TestContainerPod.Status.PodIP) // the clientIP returned by agnhost contains port if !strings.HasPrefix(clientIP, config.TestContainerPod.Status.PodIP) { framework.Failf("Source IP was NOT preserved") } } }) ginkgo.It("should only target nodes with endpoints", func() { namespace := f.Namespace.Name serviceName := "external-local-nodes" jig := e2eservice.NewTestJig(cs, namespace, serviceName) nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests) framework.ExpectNoError(err) svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, false, func(svc *v1.Service) { // Change service port to avoid collision with opened hostPorts // in other tests that run in parallel. if len(svc.Spec.Ports) != 0 { svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(svc.Spec.Ports[0].Port)) svc.Spec.Ports[0].Port = 8081 } }) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc)) defer func() { err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout) framework.ExpectNoError(err) err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() healthCheckNodePort := int(svc.Spec.HealthCheckNodePort) if healthCheckNodePort == 0 { framework.Failf("Service HealthCheck NodePort was not allocated") } ips := e2enode.CollectAddresses(nodes, v1.NodeInternalIP) ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) svcTCPPort := int(svc.Spec.Ports[0].Port) const threshold = 2 config := e2enetwork.NewNetworkingTestConfig(f) for i := 0; i < len(nodes.Items); i++ { endpointNodeName := nodes.Items[i].Name ginkgo.By("creating a pod to be part of the service " + serviceName + " on node " + endpointNodeName) _, err = jig.Run(func(rc *v1.ReplicationController) { rc.Name = serviceName if endpointNodeName != "" { rc.Spec.Template.Spec.NodeName = endpointNodeName } }) framework.ExpectNoError(err) ginkgo.By(fmt.Sprintf("waiting for service endpoint on node %v", endpointNodeName)) err = jig.WaitForEndpointOnNode(endpointNodeName) framework.ExpectNoError(err) // HealthCheck should pass only on the node where num(endpoints) > 0 // All other nodes should fail the healthcheck on the service healthCheckNodePort for n, internalIP := range ips { // Make sure the loadbalancer picked up the health check change. // Confirm traffic can reach backend through LB before checking healthcheck nodeport. e2eservice.TestReachableHTTP(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout) expectedSuccess := nodes.Items[n].Name == endpointNodeName port := strconv.Itoa(healthCheckNodePort) ipPort := net.JoinHostPort(internalIP, port) framework.Logf("Health checking %s, http://%s/healthz, expectedSuccess %v", nodes.Items[n].Name, ipPort, expectedSuccess) err := testHTTPHealthCheckNodePortFromTestContainer( config, internalIP, healthCheckNodePort, e2eservice.KubeProxyEndpointLagTimeout, expectedSuccess, threshold) framework.ExpectNoError(err) } framework.ExpectNoError(e2erc.DeleteRCAndWaitForGC(f.ClientSet, namespace, serviceName)) } }) ginkgo.It("should work from pods", func() { var err error namespace := f.Namespace.Name serviceName := "external-local-pods" jig := e2eservice.NewTestJig(cs, namespace, serviceName) svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc)) defer func() { err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout) framework.ExpectNoError(err) err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) port := strconv.Itoa(int(svc.Spec.Ports[0].Port)) ipPort := net.JoinHostPort(ingressIP, port) path := fmt.Sprintf("%s/clientip", ipPort) ginkgo.By("Creating pause pod deployment to make sure, pausePods are in desired state") deployment := createPausePodDeployment(cs, "pause-pod-deployment", namespace, 1) framework.ExpectNoError(e2edeployment.WaitForDeploymentComplete(cs, deployment), "Failed to complete pause pod deployment") defer func() { framework.Logf("Deleting deployment") err = cs.AppsV1().Deployments(namespace).Delete(context.TODO(), deployment.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err, "Failed to delete deployment %s", deployment.Name) }() deployment, err = cs.AppsV1().Deployments(namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Error in retrieving pause pod deployment") labelSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) framework.ExpectNoError(err, "Error in setting LabelSelector as selector from deployment") pausePods, err := cs.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector.String()}) framework.ExpectNoError(err, "Error in listing pods associated with pause pod deployments") pausePod := pausePods.Items[0] framework.Logf("Waiting up to %v curl %v", e2eservice.KubeProxyLagTimeout, path) cmd := fmt.Sprintf(`curl -q -s --connect-timeout 30 %v`, path) var srcIP string loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs) ginkgo.By(fmt.Sprintf("Hitting external lb %v from pod %v on node %v", ingressIP, pausePod.Name, pausePod.Spec.NodeName)) if pollErr := wait.PollImmediate(framework.Poll, loadBalancerPropagationTimeout, func() (bool, error) { stdout, err := framework.RunHostCmd(pausePod.Namespace, pausePod.Name, cmd) if err != nil { framework.Logf("got err: %v, retry until timeout", err) return false, nil } srcIP = strings.TrimSpace(strings.Split(stdout, ":")[0]) return srcIP == pausePod.Status.PodIP, nil }); pollErr != nil { framework.Failf("Source IP not preserved from %v, expected '%v' got '%v'", pausePod.Name, pausePod.Status.PodIP, srcIP) } }) ginkgo.It("should handle updates to ExternalTrafficPolicy field", func() { namespace := f.Namespace.Name serviceName := "external-local-update" jig := e2eservice.NewTestJig(cs, namespace, serviceName) nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests) framework.ExpectNoError(err) if len(nodes.Items) < 2 { framework.Failf("Need at least 2 nodes to verify source ip from a node without endpoint") } svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil) framework.ExpectNoError(err) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc)) defer func() { err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout) framework.ExpectNoError(err) err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }() // save the health check node port because it disappears when ESIPP is turned off. healthCheckNodePort := int(svc.Spec.HealthCheckNodePort) ginkgo.By("turning ESIPP off") svc, err = jig.UpdateService(func(svc *v1.Service) { svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster }) framework.ExpectNoError(err) if svc.Spec.HealthCheckNodePort > 0 { framework.Failf("Service HealthCheck NodePort still present") } epNodes, err := jig.ListNodesWithEndpoint() framework.ExpectNoError(err) // map from name of nodes with endpoint to internal ip // it is assumed that there is only a single node with the endpoint endpointNodeMap := make(map[string]string) // map from name of nodes without endpoint to internal ip noEndpointNodeMap := make(map[string]string) for _, node := range epNodes { ips := e2enode.GetAddresses(&node, v1.NodeInternalIP) if len(ips) < 1 { framework.Failf("No internal ip found for node %s", node.Name) } endpointNodeMap[node.Name] = ips[0] } for _, n := range nodes.Items { ips := e2enode.GetAddresses(&n, v1.NodeInternalIP) if len(ips) < 1 { framework.Failf("No internal ip found for node %s", n.Name) } if _, ok := endpointNodeMap[n.Name]; !ok { noEndpointNodeMap[n.Name] = ips[0] } } framework.ExpectNotEqual(len(endpointNodeMap), 0) framework.ExpectNotEqual(len(noEndpointNodeMap), 0) svcTCPPort := int(svc.Spec.Ports[0].Port) svcNodePort := int(svc.Spec.Ports[0].NodePort) ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0]) path := "/clientip" dialCmd := "clientip" config := e2enetwork.NewNetworkingTestConfig(f) ginkgo.By(fmt.Sprintf("endpoints present on nodes %v, absent on nodes %v", endpointNodeMap, noEndpointNodeMap)) for nodeName, nodeIP := range noEndpointNodeMap { ginkgo.By(fmt.Sprintf("Checking %v (%v:%v/%v) proxies to endpoints on another node", nodeName, nodeIP[0], svcNodePort, dialCmd)) _, err := GetHTTPContentFromTestContainer(config, nodeIP, svcNodePort, e2eservice.KubeProxyLagTimeout, dialCmd) framework.ExpectNoError(err, "Could not reach HTTP service through %v:%v/%v after %v", nodeIP, svcNodePort, dialCmd, e2eservice.KubeProxyLagTimeout) } for nodeName, nodeIP := range endpointNodeMap { ginkgo.By(fmt.Sprintf("checking kube-proxy health check fails on node with endpoint (%s), public IP %s", nodeName, nodeIP)) var body string pollFn := func() (bool, error) { // we expect connection failure here, but not other errors resp, err := config.GetResponseFromTestContainer( "http", "healthz", nodeIP, healthCheckNodePort) if err != nil { return false, nil } if len(resp.Errors) > 0 { return true, nil } if len(resp.Responses) > 0 { body = resp.Responses[0] } return false, nil } if pollErr := wait.PollImmediate(framework.Poll, e2eservice.TestTimeout, pollFn); pollErr != nil { framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. body %s", nodeName, healthCheckNodePort, body) } } // Poll till kube-proxy re-adds the MASQUERADE rule on the node. ginkgo.By(fmt.Sprintf("checking source ip is NOT preserved through loadbalancer %v", ingressIP)) var clientIP string pollErr := wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) { clientIP, err := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip") if err != nil { return false, nil } if strings.HasPrefix(clientIP, "10.") { return true, nil } return false, nil }) if pollErr != nil { framework.Failf("Source IP WAS preserved even after ESIPP turned off. Got %v, expected a ten-dot cluster ip.", clientIP) } // TODO: We need to attempt to create another service with the previously // allocated healthcheck nodePort. If the health check nodePort has been // freed, the new service creation will succeed, upon which we cleanup. // If the health check nodePort has NOT been freed, the new service // creation will fail. ginkgo.By("setting ExternalTraffic field back to OnlyLocal") svc, err = jig.UpdateService(func(svc *v1.Service) { svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal // Request the same healthCheckNodePort as before, to test the user-requested allocation path svc.Spec.HealthCheckNodePort = int32(healthCheckNodePort) }) framework.ExpectNoError(err) pollErr = wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) { clientIP, err := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, path) if err != nil { return false, nil } ginkgo.By(fmt.Sprintf("Endpoint %v:%v%v returned client ip %v", ingressIP, svcTCPPort, path, clientIP)) if !strings.HasPrefix(clientIP, "10.") { return true, nil } return false, nil }) if pollErr != nil { framework.Failf("Source IP (%v) is not the client IP even after ESIPP turned on, expected a public IP.", clientIP) } }) }) </del> // execAffinityTestForSessionAffinityTimeout is a helper function that wrap the logic of // affinity test for non-load-balancer services. Session afinity will be // enabled when the service is created and a short timeout will be configured so

Discussion:
Which jobs are flaking: gce-master-scale-correctness Which test(s) are flaking: Services should be able to change the type and ports of a service [Slow] Testgrid link: https://k8s- Reason for failure: Test timeouts in ~half of the runs since it was enabled here: /sig scalability /sig network
/cc /cc
The context is - the problems are strictly related to scale - 5k-node jobs are timing out. I very briefly skimmed through the test and didn't see anything obviously suspicious (apart from the fact there are gazillions long timeouts there, which alltogether sum up long hours). That said - it seems to me that there are effectively two separate tests inside it: one for TCP one for UDP They don't seem to share anything with each other. - would it make sense to split this test two effectively two separate tests: one for TCP and one for UDP. (I guess it would also allow us to share some code there).
[And potentially it could help with debugging the problem if it is strictly related to only one of those cases...]
I wanted to focus on this past week but I didn´t have time, this is my list of more expensive times and approx average duration, all of the sig-network ones use LoadBalancers. I wanted to time the test to understand if there are places where can save time ... I don´t have much context on those tests, maybe Tim can help us more, but this comment makes me think that the use of both protocols is intended
/triage accepted
/triage accepted It's been a very long time since I touched this, and I recall holding my nose on it in the first place. Splitting TCP from UDP seems reasonable if we can run them in parallel. we'll want to share the flow, and parameterize some shared funcs. I don't know that this will prevent test timeouts, though.
We could use a volunteer to go poke at this, and see how hard it will really be.
You mean "forcing them to run in parallel"? Or "allowing them to run in parallel"? The later doesn't require any additiona work. The former I think mean that it will still have to be a single test (just the code will be organized better) and that won't really help with debugging (nor preventing timeouts).

this was discussed yesterday in sig-net meeting and my understanding was to split current test in 2 different so the test can run in parallel, if you want to do so and one protocol doesn´t have to wait for the other. I think that is what you meant here
As long as we CAN parallelize. Creating LBs can be sloooooow. :P Antonio Ojea <:
Reopening - I don't think just splitting will address the problem. It may help us with debugging though...
With the split we moved from 60 mins to run the test, to run 2 tests in [30-40] mins each https://k8s- Seems a win :smile:
Awesome - that's something I was silently counting for TBH :) [I doubt it will fix flakiness, but maybe it will make it easier to debug...]
Actually - I think we're getting close thanks to the two changes ( + this one). Here is the run: This particular one filed on UDP (not sure if it's just UDP-related, or just manifested with UDP now). And we have a clear failure now: In my PR I decreased the timeout from 2h to 45m, as it seemed to me that in the past it should be enough. But it failed on 45m timeout here. So I think the next question now is: is it that slow - that we need more than 45m or something isn't working as expected (either in the k8s itself or in the test) But I think we have a clear single place to look at now.
Actually - the full stacktrace: So it's this line that failed: What is interesting is that something was changing during those 45 minutes:
Note that it's 4th time we actually wait for LB in this test. The failure happens after the pod were scaled down to 0 and then to 1. So that makes me think, that's it's actually not LB-related, but rather something internally in k8s, becuase L4 LB doesn't depend on pods - it targets nodes...
If it is always UDP, I start to see ghosts between GCE and UDP ... we only have KIND to compare against, or do we have some other non gce enviroment?

I mean, there are other tests failing in UDP , and show much more failures in GCE than in kind
- this does not look "normal". Can we give some prio to helping track this flake?
hmm interesting. we will investigate.
/assign
- any progress on this?
I haven't had a chance to look into this yet. I will prioritize it this week.
Digging through the logs I believe this was a bug in the test code that was fixed by . In the test, the service port changes around 12:33. However due to the bug, the test continues to use the old service port as seen in the test output: The port should be 81. I believe the following sequence occurs from this point: 1) The reachability test fails in the beginning because the service has changed the port and the load balancer has not been updated yet resulting in the error. 1) Around 12:38, the service controller is syncing the service and notices that the service port has changed and starts to delete the forwarding rule causing the error to change to . 1) Once the forwarding rule is recreated, the error goes back to being because the forwarding rule exists but the wrong port is used. Looking at , the test runs after the one linked have all passed. Is there anything else that needs to be investigated as part of this issue?
OK - the test was quite flaky in the past but indeed it didn't flake for the last 3 weeks (neither this nor TCP one). I'm fine with closing this one - we can open a new one if this happens again. - thanks for debugging. - it seems that the splitting of the test actually made things waay better. Which is great.