Change:
What would you like to be : The test currently only works on Cloud Providers or environments that set the External IP on nodes, because it is using the framework directly to check the IPs of the pods, using a no-snat-test-proxy to check if the Pods are not using SNAT. However, this can be simplified, removing the no-snat-test-proxy Pod and executing the queries directly in a Pod. Why is this needed: The test should work in all environments, and it's failing https://k8s-,%20master&include-filter-by-regex=NoSNAT
/sig network /help wanted /failing test
/remove kind-feature /kind failing-test
/help
This request has been marked as needing help from a contributor. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /help Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/remove kind-feature
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry. ‍
I would like to investigate this problem. /assign
I have filed , how could I verify if my patch fixes the problem?
I run them locally with KIND

Discussion:
import ( "context" "fmt" <del> "io/ioutil" "net/http" "strconv" "strings" </del> <ins> "net" </ins> "time" <ins> "github.com/onsi/ginkgo" </ins> v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" <del> "k8s.io/kubernetes/test/e2e/framework" "github.com/onsi/ginkgo" </del> <ins> "k8s.io/kubernetes/test/e2e/framework" </ins> imageutils "k8s.io/kubernetes/test/utils/image" ) const ( <del> testPodPort = 8080 testProxyPort = 31235 // Firewall rule allows external traffic on ports 30000-32767. I just picked a random one. </del> <ins> testPodPort    = "8080" noSNATTestName = "no-snat-test" </ins> ) var ( testPod = v1.Pod{ ObjectMeta: metav1.ObjectMeta{ <del> GenerateName: "no-snat-test", </del> <ins> GenerateName: noSNATTestName, </ins> Labels: map[string]string{ <del> "no-snat-test": "", </del> <ins> noSNATTestName: "", </ins> }, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { <del> Name:  "no-snat-test", </del> <ins> Name:  noSNATTestName, </ins> Image: imageutils.GetE2EImage(imageutils.Agnhost), <del> Args:  []string{"no-snat-test", "--port", strconv.Itoa(testPodPort)}, Env: []v1.EnvVar{ { Name:      "POD_IP", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "status.podIP"}}, }, }, }, }, }, } testProxyPod = v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "no-snat-test-proxy", }, Spec: v1.PodSpec{ HostNetwork: true, Containers: []v1.Container{ { Name:  "no-snat-test-proxy", Image: imageutils.GetE2EImage(imageutils.Agnhost), Args:  []string{"no-snat-test-proxy", "--port", strconv.Itoa(testProxyPort)}, Ports: []v1.ContainerPort{ { ContainerPort: testProxyPort, HostPort:      testProxyPort, }, }, </del> <ins> Args:  []string{"netexec", "--http-port", testPodPort}, </ins> }, }, }, } ) <del> // Produces a pod spec that passes nip as NODE_IP env var using downward API func newTestPod(nodename string, nip string) *v1.Pod { pod := testPod nodeIP := v1.EnvVar{ Name:  "NODE_IP", Value: nip, } pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, nodeIP) pod.Spec.NodeName = nodename return &pod } func newTestProxyPod(nodename string) *v1.Pod { pod := testProxyPod pod.Spec.NodeName = nodename return &pod } func getIP(iptype v1.NodeAddressType, node *v1.Node) (string, error) { for _, addr := range node.Status.Addresses { if addr.Type == iptype { return addr.Address, nil } } return "", fmt.Errorf("did not find %s on Node", iptype) } func getSchedulable(nodes []v1.Node) (*v1.Node, error) { for _, node := range nodes { if !node.Spec.Unschedulable { return &node, nil } } return nil, fmt.Errorf("all Nodes were unschedulable") } func checknosnatURL(proxy, pip string, ips []string) string { return fmt.Sprintf("http://%s/checknosnat?target=%s&ips=%s", proxy, pip, strings.Join(ips, ",")) } </del> // This test verifies that a Pod on each node in a cluster can talk to Pods on every other node without SNAT. // We use the [Feature:NoSNAT] tag so that most jobs will skip this test by default. var _ = SIGDescribe("NoSNAT [Feature:NoSNAT] [Slow]", func() {

framework.ExpectNotEqual(len(nodes.Items), 0, "no Nodes in the cluster") for _, node := range nodes.Items { <del> // find the Node's internal ip address to feed to the Pod inIP, err := getIP(v1.NodeInternalIP, &node) framework.ExpectNoError(err) // target Pod at Node and feed Pod Node's InternalIP pod := newTestPod(node.Name, inIP) _, err = pc.Create(context.TODO(), pod, metav1.CreateOptions{}) </del> <ins> // target Pod at Node testPod.Spec.NodeName = node.Name _, err = pc.Create(context.TODO(), &testPod, metav1.CreateOptions{}) </ins> framework.ExpectNoError(err) } <del> // In some (most?) scenarios, the test harness doesn't run in the same network as the Pods, // which means it can't query Pods using their cluster-internal IPs. To get around this, // we create a Pod in a Node's host network, and have that Pod serve on a specific port of that Node. // We can then ask this proxy Pod to query the internal endpoints served by the test Pods. // Find the first schedulable node; masters are marked unschedulable. We don't put the proxy on the master // because in some (most?) deployments firewall rules don't allow external traffic to hit ports 30000-32767 // on the master, but do allow this on the nodes. node, err := getSchedulable(nodes.Items) framework.ExpectNoError(err) ginkgo.By("creating a no-snat-test-proxy Pod on Node " + node.Name + " port " + strconv.Itoa(testProxyPort) + " so we can target our test Pods through this Node's ExternalIP") extIP, err := getIP(v1.NodeExternalIP, node) framework.ExpectNoError(err) proxyNodeIP := extIP + ":" + strconv.Itoa(testProxyPort) _, err = pc.Create(context.TODO(), newTestProxyPod(node.Name), metav1.CreateOptions{}) framework.ExpectNoError(err) </del> ginkgo.By("waiting for all of the no-snat-test pods to be scheduled and running") err = wait.PollImmediate(10*time.Second, 1*time.Minute, func() (bool, error) { <del> pods, err := pc.List(context.TODO(), metav1.ListOptions{LabelSelector: "no-snat-test"}) </del> <ins> pods, err := pc.List(context.TODO(), metav1.ListOptions{LabelSelector: noSNATTestName}) </ins> if err != nil { return false, err }

}) framework.ExpectNoError(err) <del> ginkgo.By("waiting for the no-snat-test-proxy Pod to be scheduled and running") err = wait.PollImmediate(10*time.Second, 1*time.Minute, func() (bool, error) { pod, err := pc.Get(context.TODO(), "no-snat-test-proxy", metav1.GetOptions{}) if err != nil { return false, err } if pod.Status.Phase != v1.PodRunning { if pod.Status.Phase != v1.PodPending { return false, fmt.Errorf("expected pod to be in phase "Pending" or "Running"") } return false, nil // pod is still pending } return true, nil // pod is running }) framework.ExpectNoError(err) </del> ginkgo.By("sending traffic from each pod to the others and checking that SNAT does not occur") <del> pods, err := pc.List(context.TODO(), metav1.ListOptions{LabelSelector: "no-snat-test"}) </del> <ins> pods, err := pc.List(context.TODO(), metav1.ListOptions{LabelSelector: noSNATTestName}) </ins> framework.ExpectNoError(err) <del> // collect pod IPs podIPs := []string{} for _, pod := range pods.Items { podIPs = append(podIPs, pod.Status.PodIP+":"+strconv.Itoa(testPodPort)) } // hit the /checknosnat endpoint on each Pod, tell each Pod to check all the other Pods </del> <ins> // hit the /clientip endpoint on every other Pods to check if source ip is preserved </ins> // this test is O(n^2) but it doesn't matter because we only run this test on small clusters (~3 nodes) <del> errs := []string{} client := http.Client{ Timeout: 5 * time.Minute, } for _, pip := range podIPs { ips := []string{} for _, ip := range podIPs { if ip == pip { </del> <ins> for _, sourcePod := range pods.Items { for _, targetPod := range pods.Items { if targetPod.Name == sourcePod.Name { </ins> continue } <del> ips = append(ips, ip) </del> <ins> targetAddr := net.JoinHostPort(targetPod.Status.PodIP, testPodPort) sourceIP, execPodIP := execSourceIPTest(sourcePod, targetAddr) ginkgo.By("Verifying the preserved source ip") framework.ExpectEqual(sourceIP, execPodIP) </ins> } <del> // hit /checknosnat on pip, via proxy resp, err := client.Get(checknosnatURL(proxyNodeIP, pip, ips)) framework.ExpectNoError(err) // check error code on the response, if 500 record the body, which will describe the error if resp.StatusCode == 500 { body, err := ioutil.ReadAll(resp.Body) framework.ExpectNoError(err) errs = append(errs, string(body)) } resp.Body.Close() } // report the errors all at the end if len(errs) > 0 { str := strings.Join(errs, "n") err := fmt.Errorf("/checknosnat failed in the following cases:n%s", str) framework.ExpectNoError(err) </del> } }) })

serviceAddress := net.JoinHostPort(serviceIP, strconv.Itoa(servicePort)) for _, pausePod := range pausePods.Items { <del> sourceIP, execPodIP := execSourceipTest(pausePod, serviceAddress) </del> <ins> sourceIP, execPodIP := execSourceIPTest(pausePod, serviceAddress) </ins> ginkgo.By("Verifying the preserved source ip") framework.ExpectEqual(sourceIP, execPodIP) }

}) }) <del> func execSourceipTest(pausePod v1.Pod, serviceAddress string) (string, string) { var err error var stdout string timeout := 2 * time.Minute framework.Logf("Waiting up to %v to get response from %s", timeout, serviceAddress) cmd := fmt.Sprintf(`curl -q -s --connect-timeout 30 %s/clientip`, serviceAddress) for start := time.Now(); time.Since(start) < timeout; time.Sleep(2 * time.Second) { stdout, err = framework.RunHostCmd(pausePod.Namespace, pausePod.Name, cmd) if err != nil { framework.Logf("got err: %v, retry until timeout", err) continue } // Need to check output because it might omit in case of error. if strings.TrimSpace(stdout) == "" { framework.Logf("got empty stdout, retry until timeout") continue } break } framework.ExpectNoError(err) // The stdout return from RunHostCmd is in this format: x.x.x.x:port or [xx:xx:xx::x]:port host, _, err := net.SplitHostPort(stdout) if err != nil { // ginkgo.Fail the test if output format is unexpected. framework.Failf("exec pod returned unexpected stdout: [%v]n", stdout) } return pausePod.Status.PodIP, host } </del> // execAffinityTestForSessionAffinityTimeout is a helper function that wrap the logic of // affinity test for non-load-balancer services. Session afinity will be // enabled when the service is created and a short timeout will be configured so

import ( "bytes" "fmt" <ins> "net" </ins> "regexp" "strings" "time"

} return false } <ins> // execSourceIPTest executes curl to access "/clientip" endpoint on target address // from given Pod to check if source ip is preserved. func execSourceIPTest(sourcePod v1.Pod, targetAddr string) (string, string) { var ( err     error stdout  string timeout = 2 * time.Minute ) framework.Logf("Waiting up to %v to get response from %s", timeout, targetAddr) cmd := fmt.Sprintf(`curl -q -s --connect-timeout 30 %s/clientip`, targetAddr) for start := time.Now(); time.Since(start) < timeout; time.Sleep(2 * time.Second) { stdout, err = framework.RunHostCmd(sourcePod.Namespace, sourcePod.Name, cmd) if err != nil { framework.Logf("got err: %v, retry until timeout", err) continue } // Need to check output because it might omit in case of error. if strings.TrimSpace(stdout) == "" { framework.Logf("got empty stdout, retry until timeout") continue } break } framework.ExpectNoError(err) // The stdout return from RunHostCmd is in this format: x.x.x.x:port or [xx:xx:xx::x]:port host, _, err := net.SplitHostPort(stdout) if err != nil { // ginkgo.Fail the test if output format is unexpected. framework.Failf("exec pod returned unexpected stdout: [%v]n", stdout) } return sourcePod.Status.PodIP, host } </ins>