Change:
Which jobs are failing: kops-aws and many other jobs Which test(s) are failing: This isn't the actual test, but the text we're consistently seeing is Since when has it been failing: Testgrid links: (ref: ) Reason for failure: TBD Anything else we need to know: Very rarely tests pass when one of the tests passes: Seems related to , that was merged about the same time:
/sig cluster-lifecycle /sig aws
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /sig cluster-lifecycle /sig aws Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/priority critical-urgent this is merge blocking
/assign /cc /cc /cc
The PR we think we want to revert
is also failing, I don't think this is aws-specific
/cc
/assign
it failed with kind too :/ with the same message
Opened to revert
/unassign /sig network /sig testing /remove-sig cluster-lifecycle
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry. ‍
This may be the actual cause PR to address But we're going with revert first to be sure we've identified the offending PR
I think I've identified the root cause
Revert brought us back to green: Awaiting root cause fix
/remove-triage unresolved
/reopen merged but we're now seeing multiple tests fail since it merged Looking at it went red between these commits:
Reopened this issue. <detailsIn response to : /reopen merged but we're now seeing multiple tests fail since it merged Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
is an attempt to fix
reverting in
is the failure cluster I suspect will go down once the revert goes in
/close Failure cluster has gone down, calling this done <img width="998" alt="Screen Shot 2020-01-10 at 9 56 17 AM" src="https://user-"
Closing this issue. <detailsIn response to : /close Failure cluster has gone down, calling this done <img width="998" alt="Screen Shot 2020-01-10 at 9 56 17 AM" src="https://user-"</details

Discussion:
package network import ( <ins> "encoding/hex" </ins> "encoding/json" "fmt" "math" <ins> "net" </ins> "strconv" "strings" "time" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> "k8s.io/apimachinery/pkg/util/wait" </ins> "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" <del> e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" </del> "k8s.io/kubernetes/test/images/agnhost/net/nat" imageutils "k8s.io/kubernetes/test/utils/image"

var _ = SIGDescribe("Network", func() { const ( <del> testDaemonHTTPPort    = 11301 testDaemonTCPPort     = 11302 timeoutSeconds        = 10 postFinTimeoutSeconds = 5 </del> <ins> testDaemonHTTPPort     = 11301 testDaemonTCPPort      = 11302 deadlineTimeoutSeconds = 10 postFinTimeoutSeconds  = 30 </ins> ) fr := framework.NewDefaultFramework("network")

zero := int64(0) <ins> // Create a pod to check the conntrack entries on the host node // It mounts the host /proc/net folder to be able to access // the nf_conntrack file with the host conntrack entries privileged := true hostExecPod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "e2e-net-exec", Namespace: fr.Namespace.Name, Labels:    map[string]string{"app": "e2e-net-exec"}, }, Spec: v1.PodSpec{ HostNetwork: true, NodeName:    clientNodeInfo.name, Containers: []v1.Container{ { Name:            "e2e-net-exec", Image:           kubeProxyE2eImage, ImagePullPolicy: "Always", Args:            []string{"pause"}, VolumeMounts: []v1.VolumeMount{ { Name:      "proc-net", MountPath: "/rootfs/proc/net", ReadOnly:  true, }, }, SecurityContext: &v1.SecurityContext{ Privileged: &privileged, }, }, }, Volumes: []v1.Volume{ { Name: "proc-net", VolumeSource: v1.VolumeSource{ HostPath: &v1.HostPathVolumeSource{ Path: "/proc/net", }, }, }, }, TerminationGracePeriodSeconds: &zero, }, } fr.PodClient().CreateSync(hostExecPod) defer fr.PodClient().DeleteSync(hostExecPod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) </ins> // Some distributions (Ubuntu 16.04 etc.) don't support the proc file. <del> _, err = e2essh.IssueSSHCommandWithResult( "ls /proc/net/nf_conntrack", framework.TestContext.Provider, clientNodeInfo.node) </del> <ins> _, err = framework.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", "ls /rootfs/proc/net/nf_conntrack") </ins> if err != nil && strings.Contains(err.Error(), "No such file or directory") { framework.Skipf("The node %s does not support /proc/net/nf_conntrack", clientNodeInfo.name) } framework.ExpectNoError(err) <ins> // Create the client and server pods </ins> clientPodSpec := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "e2e-net-client",

Image:           kubeProxyE2eImage, ImagePullPolicy: "Always", Args: []string{ <del> "net", "--serve", fmt.Sprintf("0.0.0.0:%d", testDaemonHTTPPort), </del> <ins> "net", "--serve", fmt.Sprintf(":%d", testDaemonHTTPPort), </ins> }, }, },

"net", "--runner", "nat-closewait-server", "--options", <del> fmt.Sprintf(`{"LocalAddr":"0.0.0.0:%v", "PostFindTimeoutSeconds":%v}`, </del> <ins> fmt.Sprintf(`{"LocalAddr":":%v", "PostFinTimeoutSeconds":%v}`, </ins> testDaemonTCPPort, postFinTimeoutSeconds), },

ginkgo.By("Make client connect") options := nat.CloseWaitClientOptions{ <del> RemoteAddr: fmt.Sprintf("%v:%v", serverNodeInfo.nodeIP, testDaemonTCPPort), TimeoutSeconds:        timeoutSeconds, PostFinTimeoutSeconds: 0, </del> <ins> RemoteAddr:            net.JoinHostPort(serverNodeInfo.nodeIP, strconv.Itoa(testDaemonTCPPort)), TimeoutSeconds:        deadlineTimeoutSeconds, PostFinTimeoutSeconds: postFinTimeoutSeconds, </ins> LeakConnection:        true, }

`'%v' 2>/dev/null`, testDaemonHTTPPort, string(jsonBytes)) <del> framework.RunHostCmdOrDie(fr.Namespace.Name, "e2e-net-client", cmd) </del> <ins> // Run the closewait command in a subroutine so it keeps waiting during postFinTimeoutSeconds // otherwise the pod is deleted and the connection is closed loosing the conntrack entry go func() { defer ginkgo.GinkgoRecover() _, err = framework.RunHostCmd(fr.Namespace.Name, "e2e-net-client", cmd) framework.ExpectNoError(err) }() </ins> <-time.After(time.Duration(1) * time.Second) ginkgo.By("Checking /proc/net/nf_conntrack for the timeout") <del> // If test flakes occur here, then this check should be performed // in a loop as there may be a race with the client connecting. e2essh.IssueSSHCommandWithResult( fmt.Sprintf("sudo cat /proc/net/nf_conntrack | grep 'dport=%v'", testDaemonTCPPort), framework.TestContext.Provider, clientNodeInfo.node) // Timeout in seconds is available as the fifth column from // /proc/net/nf_conntrack. result, err := e2essh.IssueSSHCommandWithResult( fmt.Sprintf( "sudo cat /proc/net/nf_conntrack "+ "| grep 'CLOSE_WAIT.*dst=%v.*dport=%v' "+ "| tail -n 1"+ "| awk '{print $5}' ", serverNodeInfo.nodeIP, testDaemonTCPPort), framework.TestContext.Provider, clientNodeInfo.node) framework.ExpectNoError(err) timeoutSeconds, err := strconv.Atoi(strings.TrimSpace(result.Stdout)) framework.ExpectNoError(err) </del> // These must be synchronized from the default values set in // pkg/apis/../defaults.go ConntrackTCPCloseWaitTimeout. The // current defaults are hidden in the initialization code. const epsilonSeconds = 60 const expectedTimeoutSeconds = 60 * 60 <del> framework.Logf("conntrack entry timeout was: %v, expected: %v", timeoutSeconds, expectedTimeoutSeconds) gomega.Expect(math.Abs(float64(timeoutSeconds - expectedTimeoutSeconds))).Should( gomega.BeNumerically("<", (epsilonSeconds))) </del> <ins> // the conntrack file uses the IPv6 expanded format ip := fullIPv6(net.ParseIP(serverNodeInfo.nodeIP)) // Obtain the corresponding conntrack entry on the host checking // the nf_conntrack file from the pod e2e-net-exec. // It retries in a loop if the entry is not found. cmd = fmt.Sprintf("cat /rootfs/proc/net/nf_conntrack "+ "| grep -m 1 'CLOSE_WAIT.*dst=%v.*dport=%v' ", ip, testDaemonTCPPort) if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) { result, err := framework.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", cmd) // retry if we can't obtain the conntrack entry if err != nil { framework.Logf("failed to obtain conntrack entry: %v %v", result, err) return false, nil } framework.Logf("conntrack entry for node %v and port %v:  %v", serverNodeInfo.nodeIP, testDaemonTCPPort, result) // Timeout in seconds is available as the fifth column of // the matched entry in /proc/net/nf_conntrack. line := strings.Fields(result) if len(line) < 5 { return false, fmt.Errorf("conntrack entry does not have a timeout field: %v", line) } timeoutSeconds, err := strconv.Atoi(line[4]) if err != nil { return false, fmt.Errorf("failed to convert matched timeout %s to integer: %v", line[4], err) } if math.Abs(float64(timeoutSeconds-expectedTimeoutSeconds)) < epsilonSeconds { return true, nil } return false, fmt.Errorf("wrong TCP CLOSE_WAIT timeout: %v expected: %v", timeoutSeconds, expectedTimeoutSeconds) }); err != nil { framework.Failf("no conntrack entry for port %d on node %s", testDaemonTCPPort, serverNodeInfo.nodeIP) } </ins> }) // Regression test for #74839, where:

} }) }) <ins> // fullIPv6 returns a string with the IP representation // if IPv6 it returns the expanded address format // credit https://stackoverflow.com/a/52003106/4532704 func fullIPv6(ip net.IP) string { if ip.To4() == nil { dst := make([]byte, hex.EncodedLen(len(ip))) _ = hex.Encode(dst, ip) return string(dst[0:4]) + ":" + string(dst[4:8]) + ":" + string(dst[8:12]) + ":" + string(dst[12:16]) + ":" + string(dst[16:20]) + ":" + string(dst[20:24]) + ":" + string(dst[24:28]) + ":" + string(dst[28:]) } return ip.String() } </ins>