Change:
What would you like to be : Right now we poll one node at a time for intra-pod tests. We ultimately fail a test if any one node is unreachable. If we instead polled all nodes, multiple times, and collected the number of times a node was unreachable, this would tell us what % of nodes on the cluster were broken, rather then just telling us the first node that was broken. not break the testing contract (were still verifying intra-pod connectivity, just doing so in a way that gives us holistic cluster info) Why is this needed: I described why this would make triage alot easier, and in general make it easier to see why tests are failing, here ... i'll collect a little more data on this to justify the decision .... as this is a conformance test so the bar for modification is higher (even if the test itself isnt changing) then regular tests.
/sig network
/assign
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry. ‍
/remove-triage unresolved

Discussion:
import ( "github.com/onsi/ginkgo" <ins> v1 "k8s.io/api/core/v1" </ins> "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"

ginkgo.Describe("Granular Checks: Pods", func() { <ins> checkNodeConnectivity := func(config *e2enetwork.NetworkingTestConfig, protocol string, port int) { // breadth first poll to quickly estimate failure. failedPodsByHost := map[string][]*v1.Pod{} // First time, we'll quickly try all pods, breadth first. for _, endpointPod := range config.EndpointPods { framework.Logf("Breadth first check of %v on host %v...", endpointPod.Status.PodIP, endpointPod.Status.HostIP) if err := config.DialFromTestContainer(protocol, endpointPod.Status.PodIP, port, 1, 0, sets.NewString(endpointPod.Name)); err != nil { if _, ok := failedPodsByHost[endpointPod.Status.HostIP]; !ok { failedPodsByHost[endpointPod.Status.HostIP] = []*v1.Pod{} } failedPodsByHost[endpointPod.Status.HostIP] = append(failedPodsByHost[endpointPod.Status.HostIP], endpointPod) framework.Logf("...failed...will try again in next pass") } } errors := []error{} // Second time, we pass through pods more carefully... framework.Logf("Going to retry %v out of %v pods....", len(failedPodsByHost), len(config.EndpointPods)) for host, failedPods := range failedPodsByHost { framework.Logf("Doublechecking %v pods in host %v which werent seen the first time.", len(failedPods), host) for _, endpointPod := range failedPods { framework.Logf("Now attempting to probe pod [[[ %v ]]]", endpointPod.Status.PodIP) if err := config.DialFromTestContainer(protocol, endpointPod.Status.PodIP, port, config.MaxTries, 0, sets.NewString(endpointPod.Name)); err != nil { errors = append(errors, err) } else { framework.Logf("Was able to reach %v on %v ", endpointPod.Status.PodIP, endpointPod.Status.HostIP) } framework.Logf("... Done probing pod [[[ %v ]]]", endpointPod.Status.PodIP) } framework.Logf("succeeded at polling %v out of %v connections", len(config.EndpointPods)-len(errors), len(config.EndpointPods)) } if len(errors) > 0 { framework.Logf("pod polling failure summary:") for _, e := range errors { framework.Logf("Collected error: %v", e) } framework.Failf("failed,  %v out of %v connections failed", len(errors), len(config.EndpointPods)) } } </ins> // Try to hit all endpoints through a test container, retry 5 times, // expect exactly one unique hostname. Each of these endpoints reports // its own hostname.

*/ framework.ConformanceIt("should function for intra-pod communication: http [NodeConformance]", func() { config := e2enetwork.NewCoreNetworkingTestConfig(f, false) <del> for _, endpointPod := range config.EndpointPods { config.DialFromTestContainer("http", endpointPod.Status.PodIP, e2enetwork.EndpointHTTPPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) } </del> <ins> checkNodeConnectivity(config, "http", e2enetwork.EndpointHTTPPort) </ins> }) /*

*/ framework.ConformanceIt("should function for intra-pod communication: udp [NodeConformance]", func() { config := e2enetwork.NewCoreNetworkingTestConfig(f, false) <del> for _, endpointPod := range config.EndpointPods { config.DialFromTestContainer("udp", endpointPod.Status.PodIP, e2enetwork.EndpointUDPPort, config.MaxTries, 0, sets.NewString(endpointPod.Name)) } </del> <ins> checkNodeConnectivity(config, "udp", e2enetwork.EndpointUDPPort) </ins> }) /*

} // DialFromEndpointContainer executes a curl via kubectl exec in an endpoint container. <del> func (config *NetworkingTestConfig) DialFromEndpointContainer(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { config.DialFromContainer(protocol, echoHostname, config.EndpointPods[0].Status.PodIP, targetIP, EndpointHTTPPort, targetPort, maxTries, minTries, expectedEps) </del> <ins> func (config *NetworkingTestConfig) DialFromEndpointContainer(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) error { return config.DialFromContainer(protocol, echoHostname, config.EndpointPods[0].Status.PodIP, targetIP, EndpointHTTPPort, targetPort, maxTries, minTries, expectedEps) </ins> } // DialFromTestContainer executes a curl via kubectl exec in a test container. <del> func (config *NetworkingTestConfig) DialFromTestContainer(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { config.DialFromContainer(protocol, echoHostname, config.TestContainerPod.Status.PodIP, targetIP, testContainerHTTPPort, targetPort, maxTries, minTries, expectedEps) </del> <ins> func (config *NetworkingTestConfig) DialFromTestContainer(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) error { return config.DialFromContainer(protocol, echoHostname, config.TestContainerPod.Status.PodIP, targetIP, testContainerHTTPPort, targetPort, maxTries, minTries, expectedEps) </ins> } // DialEchoFromTestContainer executes a curl via kubectl exec in a test container. The response is expected to match the echoMessage. <del> func (config *NetworkingTestConfig) DialEchoFromTestContainer(protocol, targetIP string, targetPort, maxTries, minTries int, echoMessage string) { </del> <ins> func (config *NetworkingTestConfig) DialEchoFromTestContainer(protocol, targetIP string, targetPort, maxTries, minTries int, echoMessage string) error { </ins> expectedResponse := sets.NewString() expectedResponse.Insert(echoMessage) var dialCommand string

} else { dialCommand = fmt.Sprintf("echo%%20%s", echoMessage) } <del> config.DialFromContainer(protocol, dialCommand, config.TestContainerPod.Status.PodIP, targetIP, testContainerHTTPPort, targetPort, maxTries, minTries, expectedResponse) </del> <ins> return config.DialFromContainer(protocol, dialCommand, config.TestContainerPod.Status.PodIP, targetIP, testContainerHTTPPort, targetPort, maxTries, minTries, expectedResponse) </ins> } // diagnoseMissingEndpoints prints debug information about the endpoints that

// maxTries == minTries will confirm that we see the expected endpoints and no // more for maxTries. Use this if you want to eg: fail a readiness check on a // pod and confirm it doesn't show up as an endpoint. <del> func (config *NetworkingTestConfig) DialFromContainer(protocol, dialCommand, containerIP, targetIP string, containerHTTPPort, targetPort, maxTries, minTries int, expectedResponses sets.String) { </del> <ins> // Returns nil if no error, or error message if failed after trying maxTries. func (config *NetworkingTestConfig) DialFromContainer(protocol, dialCommand, containerIP, targetIP string, containerHTTPPort, targetPort, maxTries, minTries int, expectedResponses sets.String) error { </ins> ipPort := net.JoinHostPort(containerIP, strconv.Itoa(containerHTTPPort)) cmd := makeCURLDialCommand(ipPort, dialCommand, protocol, targetIP, targetPort)

// Check against i+1 so we exit if minTries == maxTries. if (responses.Equal(expectedResponses) || responses.Len() == 0 && expectedResponses.Len() == 0) && i+1 >= minTries { <del> return </del> <ins> framework.Logf("reached %v after %v/%v tries", targetIP, i, maxTries) return nil </ins> } // TODO: get rid of this delay #36281 time.Sleep(hitEndpointRetryDelay) } if dialCommand == echoHostname { config.diagnoseMissingEndpoints(responses) } <del> framework.Failf("Failed to find expected responses:nTries %dnCommand %vnretrieved %vnexpected %vn", maxTries, cmd, responses, expectedResponses) </del> <ins> returnMsg := fmt.Errorf("did not find expected responses... nTries %dnCommand %vnretrieved %vnexpected %v", maxTries, cmd, responses, expectedResponses) framework.Logf("encountered error during dial (%v)", returnMsg) return returnMsg </ins> } // GetEndpointsFromTestContainer executes a curl via kubectl exec in a test container.

epCount := len(config.EndpointPods) config.MaxTries = epCount*epCount + testTries <ins> framework.Logf("Setting MaxTries for pod polling to %v for networking test based on endpoint count %v", config.MaxTries, epCount) </ins> } // setup includes setupCore and also sets up services