Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Two services referring the same endpoint. One of the services was deleted. This brought down the connectivity of the other service. What you expected to happen: The connectivity to service should continue to work How to reproduce it (as minimally and precisely as possible): two services referencing same endpoint one of the services connectivity is broken to the other service Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig network These SIGs are my best guesses for this issue. Please comment if I am incorrect about one. I am a bot run by vllry. ‍
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry. ‍
Do you mean a service referencing the same pod, or literally the same Endpoints object?
I meant service referencing the same pod.
/assign
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry. ‍
Is this still an issue? If so, it would be helpful to get an answer to Jordan's question above. If this Service is referencing the same Endpoints object that would be a very different issue than if it was referencing the same Pod. /triage needs-information
This issue happens when multiple Services are referencing a same POD. I mean the same backend IP address but different ports. i.e. the POD is hosting two workloads on different ports.
/sig windows
Is this only observed in windows environments?
Yes this happens only on Windows environments
Thanks can you provide more information about the version of Kubernetes you're using? Do you know what proxier mode kube-proxy is running with? Maybe winkernel? I'm not as familiar with the Windows implementation here, adding to help debug this. /assign
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry. ‍

Discussion:
Log(ep, "Endpoint Cleanup", 3) if ep.refCount != nil { *ep.refCount-- <del> } // Remove the remote hns endpoint, if no service is referring it // Never delete a Local Endpoint. Local Endpoints are already created by other entities. // Remove only remote endpoints created by this service if (ep.refCount == nil || *ep.refCount <= 0) && !ep.GetIsLocal() { klog.V(4).Infof("Removing endpoints for %v, since no one is referencing it", ep) err := ep.hns.deleteEndpoint(ep.hnsID) if err == nil { ep.hnsID = "" } else { klog.Errorf("Endpoint deletion failed for %v: %v", ep.IP(), err) </del> <ins> // Remove the remote hns endpoint, if no service is referring it // Never delete a Local Endpoint. Local Endpoints are already created by other entities. // Remove only remote endpoints created by this service if *ep.refCount <= 0 && !ep.GetIsLocal() { klog.V(4).Infof("Removing endpoints for %v, since no one is referencing it", ep) err := ep.hns.deleteEndpoint(ep.hnsID) if err == nil { ep.hnsID = "" } else { klog.Errorf("Endpoint deletion failed for %v: %v", ep.IP(), err) } </ins> } <ins> ep.refCount = nil </ins> } }

t.Errorf("Global refCount: %v does not match endpoint refCount: %v", *proxier.endPointsRefCount[guid], *epInfo.refCount) } } <ins> func TestSharedRemoteEndpointDelete(t *testing.T) { syncPeriod := 30 * time.Second proxier := NewFakeProxier(syncPeriod, syncPeriod, clusterCIDR, "testhost", net.ParseIP("10.0.0.1"), "L2Bridge", false) if proxier == nil { t.Error() } svcIP1 := "10.20.30.41" svcPort1 := 80 svcNodePort1 := 3001 svcPortName1 := proxy.ServicePortName{ NamespacedName: makeNSN("ns1", "svc1"), Port:           "p80", Protocol:       v1.ProtocolTCP, } svcIP2 := "10.20.30.42" svcPort2 := 80 svcNodePort2 := 3002 svcPortName2 := proxy.ServicePortName{ NamespacedName: makeNSN("ns1", "svc2"), Port:           "p80", Protocol:       v1.ProtocolTCP, } makeServiceMap(proxier, makeTestService(svcPortName1.Namespace, svcPortName1.Name, func(svc *v1.Service) { svc.Spec.Type = "NodePort" svc.Spec.ClusterIP = svcIP1 svc.Spec.Ports = []v1.ServicePort{{ Name:     svcPortName1.Port, Port:     int32(svcPort1), Protocol: v1.ProtocolTCP, NodePort: int32(svcNodePort1), }} }), makeTestService(svcPortName2.Namespace, svcPortName2.Name, func(svc *v1.Service) { svc.Spec.Type = "NodePort" svc.Spec.ClusterIP = svcIP2 svc.Spec.Ports = []v1.ServicePort{{ Name:     svcPortName2.Port, Port:     int32(svcPort2), Protocol: v1.ProtocolTCP, NodePort: int32(svcNodePort2), }} }), ) makeEndpointsMap(proxier, makeTestEndpoints(svcPortName1.Namespace, svcPortName1.Name, func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{{ Addresses: []v1.EndpointAddress{{ IP: epIpAddressRemote, }}, Ports: []v1.EndpointPort{{ Name:     svcPortName1.Port, Port:     int32(svcPort1), Protocol: v1.ProtocolTCP, }}, }} }), makeTestEndpoints(svcPortName2.Namespace, svcPortName2.Name, func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{{ Addresses: []v1.EndpointAddress{{ IP: epIpAddressRemote, }}, Ports: []v1.EndpointPort{{ Name:     svcPortName2.Port, Port:     int32(svcPort2), Protocol: v1.ProtocolTCP, }}, }} }), ) proxier.setInitialized(true) proxier.syncProxyRules() ep := proxier.endpointsMap[svcPortName1][0] epInfo, ok := ep.(*endpointsInfo) if !ok { t.Errorf("Failed to cast endpointsInfo %q", svcPortName1.String()) } else { if epInfo.hnsID != guid { t.Errorf("%v does not match %v", epInfo.hnsID, guid) } } if *proxier.endPointsRefCount[guid] != 2 { t.Errorf("RefCount not incremented. Current value: %v", *proxier.endPointsRefCount[guid]) } if *proxier.endPointsRefCount[guid] != *epInfo.refCount { t.Errorf("Global refCount: %v does not match endpoint refCount: %v", *proxier.endPointsRefCount[guid], *epInfo.refCount) } proxier.setInitialized(false) deleteServices(proxier, makeTestService(svcPortName2.Namespace, svcPortName2.Name, func(svc *v1.Service) { svc.Spec.Type = "NodePort" svc.Spec.ClusterIP = svcIP2 svc.Spec.Ports = []v1.ServicePort{{ Name:     svcPortName2.Port, Port:     int32(svcPort2), Protocol: v1.ProtocolTCP, NodePort: int32(svcNodePort2), }} }), ) deleteEndpoints(proxier, makeTestEndpoints(svcPortName2.Namespace, svcPortName2.Name, func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{{ Addresses: []v1.EndpointAddress{{ IP: epIpAddressRemote, }}, Ports: []v1.EndpointPort{{ Name:     svcPortName2.Port, Port:     int32(svcPort2), Protocol: v1.ProtocolTCP, }}, }} }), ) proxier.setInitialized(true) proxier.syncProxyRules() ep = proxier.endpointsMap[svcPortName1][0] epInfo, ok = ep.(*endpointsInfo) if !ok { t.Errorf("Failed to cast endpointsInfo %q", svcPortName1.String()) } else { if epInfo.hnsID != guid { t.Errorf("%v does not match %v", epInfo.hnsID, guid) } } if *epInfo.refCount != 1 { t.Errorf("Incorrect Refcount. Current value: %v", *epInfo.refCount) } if *proxier.endPointsRefCount[guid] != *epInfo.refCount { t.Errorf("Global refCount: %v does not match endpoint refCount: %v", *proxier.endPointsRefCount[guid], *epInfo.refCount) } } func TestSharedRemoteEndpointUpdate(t *testing.T) { syncPeriod := 30 * time.Second proxier := NewFakeProxier(syncPeriod, syncPeriod, clusterCIDR, "testhost", net.ParseIP("10.0.0.1"), "L2Bridge", false) if proxier == nil { t.Error() } svcIP1 := "10.20.30.41" svcPort1 := 80 svcNodePort1 := 3001 svcPortName1 := proxy.ServicePortName{ NamespacedName: makeNSN("ns1", "svc1"), Port:           "p80", Protocol:       v1.ProtocolTCP, } svcIP2 := "10.20.30.42" svcPort2 := 80 svcNodePort2 := 3002 svcPortName2 := proxy.ServicePortName{ NamespacedName: makeNSN("ns1", "svc2"), Port:           "p80", Protocol:       v1.ProtocolTCP, } makeServiceMap(proxier, makeTestService(svcPortName1.Namespace, svcPortName1.Name, func(svc *v1.Service) { svc.Spec.Type = "NodePort" svc.Spec.ClusterIP = svcIP1 svc.Spec.Ports = []v1.ServicePort{{ Name:     svcPortName1.Port, Port:     int32(svcPort1), Protocol: v1.ProtocolTCP, NodePort: int32(svcNodePort1), }} }), makeTestService(svcPortName2.Namespace, svcPortName2.Name, func(svc *v1.Service) { svc.Spec.Type = "NodePort" svc.Spec.ClusterIP = svcIP2 svc.Spec.Ports = []v1.ServicePort{{ Name:     svcPortName2.Port, Port:     int32(svcPort2), Protocol: v1.ProtocolTCP, NodePort: int32(svcNodePort2), }} }), ) makeEndpointsMap(proxier, makeTestEndpoints(svcPortName1.Namespace, svcPortName1.Name, func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{{ Addresses: []v1.EndpointAddress{{ IP: epIpAddressRemote, }}, Ports: []v1.EndpointPort{{ Name:     svcPortName1.Port, Port:     int32(svcPort1), Protocol: v1.ProtocolTCP, }}, }} }), makeTestEndpoints(svcPortName2.Namespace, svcPortName2.Name, func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{{ Addresses: []v1.EndpointAddress{{ IP: epIpAddressRemote, }}, Ports: []v1.EndpointPort{{ Name:     svcPortName2.Port, Port:     int32(svcPort2), Protocol: v1.ProtocolTCP, }}, }} }), ) proxier.setInitialized(true) proxier.syncProxyRules() ep := proxier.endpointsMap[svcPortName1][0] epInfo, ok := ep.(*endpointsInfo) if !ok { t.Errorf("Failed to cast endpointsInfo %q", svcPortName1.String()) } else { if epInfo.hnsID != guid { t.Errorf("%v does not match %v", epInfo.hnsID, guid) } } if *proxier.endPointsRefCount[guid] != 2 { t.Errorf("RefCount not incremented. Current value: %v", *proxier.endPointsRefCount[guid]) } if *proxier.endPointsRefCount[guid] != *epInfo.refCount { t.Errorf("Global refCount: %v does not match endpoint refCount: %v", *proxier.endPointsRefCount[guid], *epInfo.refCount) } proxier.setInitialized(false) proxier.OnServiceUpdate( makeTestService(svcPortName1.Namespace, svcPortName1.Name, func(svc *v1.Service) { svc.Spec.Type = "NodePort" svc.Spec.ClusterIP = svcIP1 svc.Spec.Ports = []v1.ServicePort{{ Name:     svcPortName1.Port, Port:     int32(svcPort1), Protocol: v1.ProtocolTCP, NodePort: int32(svcNodePort1), }} }), makeTestService(svcPortName1.Namespace, svcPortName1.Name, func(svc *v1.Service) { svc.Spec.Type = "NodePort" svc.Spec.ClusterIP = svcIP1 svc.Spec.Ports = []v1.ServicePort{{ Name:     svcPortName1.Port, Port:     int32(svcPort1), Protocol: v1.ProtocolTCP, NodePort: int32(3003), }} })) proxier.OnEndpointsUpdate( makeTestEndpoints(svcPortName1.Namespace, svcPortName1.Name, func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{{ Addresses: []v1.EndpointAddress{{ IP: epIpAddressRemote, }}, Ports: []v1.EndpointPort{{ Name:     svcPortName1.Port, Port:     int32(svcPort1), Protocol: v1.ProtocolTCP, }}, }} }), makeTestEndpoints(svcPortName1.Namespace, svcPortName1.Name, func(ept *v1.Endpoints) { ept.Subsets = []v1.EndpointSubset{{ Addresses: []v1.EndpointAddress{{ IP: epIpAddressRemote, }}, Ports: []v1.EndpointPort{ { Name:     svcPortName1.Port, Port:     int32(svcPort1), Protocol: v1.ProtocolTCP, }, { Name:     "p443", Port:     int32(443), Protocol: v1.ProtocolTCP, }}, }} })) proxier.mu.Lock() proxier.endpointsSynced = true proxier.mu.Unlock() proxier.setInitialized(true) proxier.syncProxyRules() ep = proxier.endpointsMap[svcPortName1][0] epInfo, ok = ep.(*endpointsInfo) if !ok { t.Errorf("Failed to cast endpointsInfo %q", svcPortName1.String()) } else { if epInfo.hnsID != guid { t.Errorf("%v does not match %v", epInfo.hnsID, guid) } } if *epInfo.refCount != 2 { t.Errorf("Incorrect refcount. Current value: %v", *epInfo.refCount) } if *proxier.endPointsRefCount[guid] != *epInfo.refCount { t.Errorf("Global refCount: %v does not match endpoint refCount: %v", *proxier.endPointsRefCount[guid], *epInfo.refCount) } } </ins> func TestCreateLoadBalancer(t *testing.T) { syncPeriod := 30 * time.Second proxier := NewFakeProxier(syncPeriod, syncPeriod, clusterCIDR, "testhost", net.ParseIP("10.0.0.1"), "Overlay", false)

defer proxier.mu.Unlock() proxier.servicesSynced = true } <ins> func deleteServices(proxier *Proxier, allServices ...*v1.Service) { for i := range allServices { proxier.OnServiceDelete(allServices[i]) } proxier.mu.Lock() defer proxier.mu.Unlock() proxier.servicesSynced = true } </ins> func makeTestService(namespace, name string, svcFunc func(*v1.Service)) *v1.Service { svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{

proxier.endpointsSynced = true } <ins> func deleteEndpoints(proxier *Proxier, allEndpoints ...*v1.Endpoints) { for i := range allEndpoints { proxier.OnEndpointsDelete(allEndpoints[i]) } proxier.mu.Lock() defer proxier.mu.Unlock() proxier.endpointsSynced = true } </ins> func makeTestEndpoints(namespace, name string, eptFunc func(*v1.Endpoints)) *v1.Endpoints { ept := &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{