Change:
What happened: When I config kube-proxy with excludeCIDRs, kube-proxy still try to delete my ipvs rules. What you expected to happen: kube-proxy not delete excludeCIDRs virturl server. How to reproduce it (as minimally and precisely as possible): kubeadm init --config=kubeadm- --experimental-upload-certs Then add a ipvs on node: Then join the node, you can see kube-proxy log: If don't have an active connection, the ipvs will removed. Environment: Kubernetes version (use ): v1.14.0 OS (e.g: ): centos7.4 Kernel (e.g. ): 10 Install tools: kubeadm
/sig network
/triage unresolved I am a bot run by ‍
Kube-proxy delete RS from VIP, even if VIP in excludeCIDRs Because RS are the same with RS from Service VIP's Expected: But kube-proxy make that: - -
I think this is not we expected. I want using ipvs as a local porxy for kubernetes HA.
I've raised the pr
/remove-triage unresolved

Discussion:
func (proxier *Proxier) cleanLegacyService(activeServices map[string]bool, currentServices map[string]*utilipvs.VirtualServer, legacyBindAddrs map[string]bool) { for cs := range currentServices { svc := currentServices[cs] <ins> if proxier.isIPInExcludeCIDRs(svc.Address) { continue } </ins> if _, ok := activeServices[cs]; !ok { <del> // This service was not processed in the latest sync loop so before deleting it, okayToDelete := true </del> rsList, _ := proxier.ipvs.GetRealServers(svc) // If we still have real servers graceful termination is not done if len(rsList) > 0 { <del> okayToDelete = false </del> <ins> continue </ins> } // Applying graceful termination to all real servers for _, rs := range rsList { uniqueRS := GetUniqueRSName(svc, rs)

klog.Errorf("Failed to delete destination: %v, error: %v", uniqueRS, err) } } <del> // make sure it does not fall within an excluded CIDR range. for _, excludedCIDR := range proxier.excludeCIDRs { // Any validation of this CIDR already should have occurred. _, n, _ := net.ParseCIDR(excludedCIDR) if n.Contains(svc.Address) { okayToDelete = false break } </del> <ins> klog.V(4).Infof("Delete service %s", svc.String()) if err := proxier.ipvs.DeleteVirtualServer(svc); err != nil { klog.Errorf("Failed to delete service %s, error: %v", svc.String(), err) </ins> } <del> if okayToDelete { klog.V(4).Infof("Delete service %s", svc.String()) if err := proxier.ipvs.DeleteVirtualServer(svc); err != nil { klog.Errorf("Failed to delete service %s, error: %v", svc.String(), err) } addr := svc.Address.String() if _, ok := legacyBindAddrs[addr]; ok { klog.V(4).Infof("Unbinding address %s", addr) if err := proxier.netlinkHandle.UnbindAddress(addr, DefaultDummyDevice); err != nil { klog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err) } else { // In case we delete a multi-port service, avoid trying to unbind multiple times delete(legacyBindAddrs, addr) } </del> <ins> addr := svc.Address.String() if _, ok := legacyBindAddrs[addr]; ok { klog.V(4).Infof("Unbinding address %s", addr) if err := proxier.netlinkHandle.UnbindAddress(addr, DefaultDummyDevice); err != nil { klog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err) } else { // In case we delete a multi-port service, avoid trying to unbind multiple times delete(legacyBindAddrs, addr) </ins> } } } } } <ins> func (proxier *Proxier) isIPInExcludeCIDRs(ip net.IP) bool { // make sure it does not fall within an excluded CIDR range. for _, excludedCIDR := range proxier.excludeCIDRs { // Any validation of this CIDR already should have occurred. _, n, _ := net.ParseCIDR(excludedCIDR) if n.Contains(ip) { return true } } return false } </ins> func (proxier *Proxier) getLegacyBindAddr(activeBindAddrs map[string]bool, currentBindAddrs []string) map[string]bool { legacyAddrs := make(map[string]bool) isIpv6 := utilnet.IsIPv6(proxier.nodeIP)

} <ins> func TestCleanLegacyRealServersExcludeCIDRs(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake() ipset := ipsettest.NewFake(testIPSetVersion) gtm := NewGracefulTerminationManager(ipvs) fp := NewFakeProxier(ipt, ipvs, ipset, nil, []string{"4.4.4.4/32"}) fp.gracefuldeleteManager = gtm vs := &utilipvs.VirtualServer{ Address:   net.ParseIP("4.4.4.4"), Protocol:  string(v1.ProtocolUDP), Port:      56, Scheduler: "rr", Flags:     utilipvs.FlagHashed, } fp.ipvs.AddVirtualServer(vs) rss := []*utilipvs.RealServer{ { Address:      net.ParseIP("10.10.10.10"), Port:         56, ActiveConn:   0, InactiveConn: 0, }, { Address:      net.ParseIP("11.11.11.11"), Port:         56, ActiveConn:   0, InactiveConn: 0, }, } for _, rs := range rss { fp.ipvs.AddRealServer(vs, rs) } fp.netlinkHandle.EnsureDummyDevice(DefaultDummyDevice) fp.netlinkHandle.EnsureAddressBind("4.4.4.4", DefaultDummyDevice) fp.cleanLegacyService( map[string]bool{}, map[string]*utilipvs.VirtualServer{"ipvs0": vs}, map[string]bool{"4.4.4.4": true}, ) fp.gracefuldeleteManager.tryDeleteRs() remainingRealServers, _ := fp.ipvs.GetRealServers(vs) if len(remainingRealServers) != 2 { t.Errorf("Expected number of remaining IPVS real servers after cleanup should be %v. Got %v", 2, len(remainingRealServers)) } } </ins> func TestCleanLegacyService6(t *testing.T) { ipt := iptablestest.NewFake() ipvs := ipvstest.NewFake()