Change:
What would you like to be : kube-proxy should expose better metrics around tracking success and failures. Administrators should be able to alert on failure counts. Why is this needed: There are several motivating cases: iptables lock contention causes to fail at an increasing rate as services + endpoints grows or are failing to execute Ultimately, there is essentially no feedback from kube-proxy if it fails to program the service network. This makes it annoyingly difficult to debug when things go wrong. As many reasonable hints we can provide should be . /sig network
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry. ‍
is anyone working one this..
what cases do you think are poorly covered? /assign
thanks for triaging this :-) The specific cases are when fails - that function logs an error but doesn't otherwise surface any information. We should have a top-level metric for failing, along with any other lower-level metrics that might make sense for administrators, e.g. fails, fails.
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry. ‍
/help-wanted
/assign

Discussion:
err = ipt.Restore(utiliptables.TableNAT, natLines, utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { klog.Errorf("Failed to execute iptables-restore for %s: %v", utiliptables.TableNAT, err) <ins> metrics.IptablesRestoreFailuresTotal.Inc() </ins> encounteredError = true } }

// Write it. if err := ipt.Restore(utiliptables.TableFilter, filterLines, utiliptables.NoFlushTables, utiliptables.RestoreCounters); err != nil { klog.Errorf("Failed to execute iptables-restore for %s: %v", utiliptables.TableFilter, err) <ins> metrics.IptablesRestoreFailuresTotal.Inc() </ins> encounteredError = true } }

err = proxier.iptables.RestoreAll(proxier.iptablesData.Bytes(), utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { klog.Errorf("Failed to execute iptables-restore: %v", err) <ins> metrics.IptablesRestoreFailuresTotal.Inc() </ins> // Revert new local ports. klog.V(2).Infof("Closing local ports after iptables-restore failure") utilproxy.RevertPorts(replacementPortsMap, proxier.portsMap)

err = proxier.iptables.RestoreAll(proxier.iptablesData.Bytes(), utiliptables.NoFlushTables, utiliptables.RestoreCounters) if err != nil { klog.Errorf("Failed to execute iptables-restore: %vnRules:n%s", err, proxier.iptablesData.Bytes()) <ins> metrics.IptablesRestoreFailuresTotal.Inc() </ins> // Revert new local ports. utilproxy.RevertPorts(replacementPortsMap, proxier.portsMap) return

Help:      "Cumulative proxy rules Service changes", }, ) <ins> // IptablesRestoreFailuresTotal is the number of iptables restore failures that the proxy has // seen. IptablesRestoreFailuresTotal = prometheus.NewCounter( prometheus.CounterOpts{ Subsystem: kubeProxySubsystem, Name:      "sync_proxy_rules_iptables_restore_failures_total", Help:      "Cumulative proxy iptables restore failures", }, ) </ins> ) var registerMetricsOnce sync.Once

prometheus.MustRegister(EndpointChangesTotal) prometheus.MustRegister(ServiceChangesPending) prometheus.MustRegister(ServiceChangesTotal) <ins> prometheus.MustRegister(IptablesRestoreFailuresTotal) </ins> }) }