Change:
<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: ci-kubernetes-e2e-gci-gce-alpha-features Which test(s) are failing: Networking should recreate its iptables rules if they are deleted [Disruptive] Since when has it been failing: Commit Testgrid link: Reason for failure: Anything else we need to know: Possibly related to this? It's listed in the git diff /sig network
/assign /cc Thanks for detecting it so quicklly
most probably is related to
/unassign /assign he has already submitted a fix
/triage accepted
Reviewed. Marking as resolved on the CI Signal Board. Once was merged, failures stopped and they have not re-occurred in the last 8 days (Oct 10th)
Yay

Discussion:
ipv6Primary := kl.nodeIP != nil && utilnet.IsIPv6(kl.nodeIP) var iptClients []utiliptables.Interface <ins> var protocols []utiliptables.Protocol </ins> if maybeDualStack || !ipv6Primary { <ins> protocols = append(protocols, utiliptables.ProtocolIPv4) </ins> iptClients = append(iptClients, utiliptables.New(exec, utiliptables.ProtocolIPv4)) } if maybeDualStack || ipv6Primary { <ins> protocols = append(protocols, utiliptables.ProtocolIPv6) </ins> iptClients = append(iptClients, utiliptables.New(exec, utiliptables.ProtocolIPv6)) } <del> for _, iptClient := range iptClients { kl.syncNetworkUtil(iptClient) go iptClient.Monitor( utiliptables.Chain("KUBE-KUBELET-CANARY"), []utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter}, func() { kl.syncNetworkUtil(iptClient) }, 1*time.Minute, wait.NeverStop, ) </del> <ins> for i := range iptClients { iptClient := iptClients[i] if kl.syncNetworkUtil(iptClient) { klog.Infof("Initialized %s iptables rules.", protocols[i]) go iptClient.Monitor( utiliptables.Chain("KUBE-KUBELET-CANARY"), []utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter}, func() { kl.syncNetworkUtil(iptClient) }, 1*time.Minute, wait.NeverStop, ) } else { klog.Warningf("Failed to initialize %s iptables rules; some functionality may be missing.", protocols[i]) } </ins> } }

// 	Marked connection will be drop on INPUT/OUTPUT Chain in filter table // 2. 	In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT // 	Marked connection will get SNAT on POSTROUTING Chain in nat table <del> func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) { </del> <ins> func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) bool { </ins> // Setup KUBE-MARK-DROP rules dropMark := getIPTablesMark(kl.iptablesDropBit) if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil { klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err) <del> return </del> <ins> return false </ins> } if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--or-mark", dropMark); err != nil { klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err) <del> return </del> <ins> return false </ins> } if _, err := iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil { klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableFilter, KubeFirewallChain, err) <del> return </del> <ins> return false </ins> } if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, "-m", "comment", "--comment", "kubernetes firewall for dropping marked packets", "-m", "mark", "--mark", fmt.Sprintf("%s/%s", dropMark, dropMark), "-j", "DROP"); err != nil { klog.Errorf("Failed to ensure rule to drop packet marked by %v in %v chain %v: %v", KubeMarkDropChain, utiliptables.TableFilter, KubeFirewallChain, err) <del> return </del> <ins> return false </ins> } // drop all non-local packets to localhost if they're not part of an existing

"!", "--ctstate", "RELATED,ESTABLISHED,DNAT", "-j", "DROP"); err != nil { klog.Errorf("Failed to ensure rule to drop invalid localhost packets in %v chain %v: %v", utiliptables.TableFilter, KubeFirewallChain, err) <del> return </del> <ins> return false </ins> } } if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil { klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainOutput, KubeFirewallChain, err) <del> return </del> <ins> return false </ins> } if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil { klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainInput, KubeFirewallChain, err) <del> return </del> <ins> return false </ins> } // Setup KUBE-MARK-MASQ rules masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit) if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil { klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkMasqChain, err) <del> return </del> <ins> return false </ins> } if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil { klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err) <del> return </del> <ins> return false </ins> } if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--or-mark", masqueradeMark); err != nil { klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err) <del> return </del> <ins> return false </ins> } if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting, "-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil { klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err) <del> return </del> <ins> return false </ins> } // Set up KUBE-POSTROUTING to unmark and masquerade marked packets

"-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", masqueradeMark, masqueradeMark), "-j", "RETURN"); err != nil { klog.Errorf("Failed to ensure filtering rule for %v: %v", KubePostroutingChain, err) <del> return </del> <ins> return false </ins> } // Clear the mark to avoid re-masquerading if the packet re-traverses the network stack. // We know the mark bit is currently set so we can use --xor-mark to clear it (without needing

if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, "-j", "MARK", "--xor-mark", masqueradeMark); err != nil { klog.Errorf("Failed to ensure unmarking rule for %v: %v", KubePostroutingChain, err) <del> return </del> <ins> return false </ins> } masqRule := []string{ "-m", "comment", "--comment", "kubernetes service traffic requiring SNAT",

} if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, masqRule...); err != nil { klog.Errorf("Failed to ensure SNAT rule for packets marked by %v in %v chain %v: %v", KubeMarkMasqChain, utiliptables.TableNAT, KubePostroutingChain, err) <del> return </del> <ins> return false </ins> } <ins> return true </ins> } // getIPTablesMark returns the fwmark given the bit

// restart iptables"?). So instead we just manually delete all "KUBE-" // chains. <del> ginkgo.By("dumping iptables rules on a node") </del> <ins> ginkgo.By("dumping iptables rules on node " + host) </ins> result, err := e2essh.SSH("sudo iptables-save", host, framework.TestContext.Provider) <ins> e2essh.LogResult(result) </ins> if err != nil || result.Code != 0 { <del> e2essh.LogResult(result) </del> framework.Failf("couldn't dump iptable rules: %v", err) }

} return false, nil }) <ins> if err != nil { e2essh.LogResult(result) } </ins> framework.ExpectNoError(err, "kubelet did not recreate its iptables rules") }) })