Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: Mixed protocols LoadBalacer services are not supported by apiserver. It will decline those services with error "cannot create an external load balancer with non-TCP/UDP ports". Hence another annotation for this purpose. e.g. However, this is still not working. The UDP LB rule is still not created by the above service. What you expected to happen: Both TCP and UDP rules created for the service. How to reproduce it (as minimally and precisely as possible): Create a service with above annotation. Anything else we need to know?: Environment: Kubernetes version (use ): 1.13.1 Cloud provider or hardware configuration: azure OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: /sig azure /kind bug /assign
I tried with the above exact service config, and it works well on my k8s v1.13.3 cluster created by aks-engine, how can I repro that error?
correction: there is no error, while UDP port rule is not created, I can repro that.

Discussion:
var expectedProbes []network.Probe var expectedRules []network.LoadBalancingRule for _, port := range ports { <del> lbRuleName := az.getLoadBalancerRuleName(service, port, subnet(service)) klog.V(2).Infof("reconcileLoadBalancerRule lb name (%s) rule name (%s)", lbName, lbRuleName) transportProto, _, probeProto, err := getProtocolsFromKubernetesProtocol(port.Protocol) if err != nil { return expectedProbes, expectedRules, err </del> <ins> protocols := []v1.Protocol{port.Protocol} if v, ok := service.Annotations[ServiceAnnotationLoadBalancerMixedProtocols]; ok && v == "true" { klog.V(2).Infof("reconcileLoadBalancerRule lb name (%s) flag(%s) is set", lbName, ServiceAnnotationLoadBalancerMixedProtocols) if port.Protocol == v1.ProtocolTCP { protocols = append(protocols, v1.ProtocolUDP) } else if port.Protocol == v1.ProtocolUDP { protocols = append(protocols, v1.ProtocolTCP) } </ins> } <del> if servicehelpers.NeedsHealthCheck(service) { podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service) </del> <ins> for _, protocol := range protocols { lbRuleName := az.getLoadBalancerRuleName(service, protocol, port.Port, subnet(service)) klog.V(2).Infof("reconcileLoadBalancerRule lb name (%s) rule name (%s)", lbName, lbRuleName) </ins> <del> expectedProbes = append(expectedProbes, network.Probe{ Name: &lbRuleName, ProbePropertiesFormat: &network.ProbePropertiesFormat{ RequestPath:       to.StringPtr(podPresencePath), Protocol:          network.ProbeProtocolHTTP, Port:              to.Int32Ptr(podPresencePort), IntervalInSeconds: to.Int32Ptr(5), NumberOfProbes:    to.Int32Ptr(2), }, }) } else if port.Protocol != v1.ProtocolUDP && port.Protocol != v1.ProtocolSCTP { // we only add the expected probe if we're doing TCP expectedProbes = append(expectedProbes, network.Probe{ Name: &lbRuleName, ProbePropertiesFormat: &network.ProbePropertiesFormat{ Protocol:          *probeProto, Port:              to.Int32Ptr(port.NodePort), IntervalInSeconds: to.Int32Ptr(5), NumberOfProbes:    to.Int32Ptr(2), }, }) } </del> <ins> transportProto, _, probeProto, err := getProtocolsFromKubernetesProtocol(protocol) if err != nil { return expectedProbes, expectedRules, err } </ins> <del> loadDistribution := network.Default if service.Spec.SessionAffinity == v1.ServiceAffinityClientIP { loadDistribution = network.SourceIP } </del> <ins> if servicehelpers.NeedsHealthCheck(service) { podPresencePath, podPresencePort := servicehelpers.GetServiceHealthCheckPathPort(service) expectedProbes = append(expectedProbes, network.Probe{ Name: &lbRuleName, ProbePropertiesFormat: &network.ProbePropertiesFormat{ RequestPath:       to.StringPtr(podPresencePath), Protocol:          network.ProbeProtocolHTTP, Port:              to.Int32Ptr(podPresencePort), IntervalInSeconds: to.Int32Ptr(5), NumberOfProbes:    to.Int32Ptr(2), }, }) } else if protocol != v1.ProtocolUDP && protocol != v1.ProtocolSCTP { // we only add the expected probe if we're doing TCP expectedProbes = append(expectedProbes, network.Probe{ Name: &lbRuleName, ProbePropertiesFormat: &network.ProbePropertiesFormat{ Protocol:          *probeProto, Port:              to.Int32Ptr(port.NodePort), IntervalInSeconds: to.Int32Ptr(5), NumberOfProbes:    to.Int32Ptr(2), }, }) } </ins> <del> expectedRule := network.LoadBalancingRule{ Name: &lbRuleName, LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ Protocol: *transportProto, FrontendIPConfiguration: &network.SubResource{ ID: to.StringPtr(lbFrontendIPConfigID), }, BackendAddressPool: &network.SubResource{ ID: to.StringPtr(lbBackendPoolID), </del> <ins> loadDistribution := network.Default if service.Spec.SessionAffinity == v1.ServiceAffinityClientIP { loadDistribution = network.SourceIP } expectedRule := network.LoadBalancingRule{ Name: &lbRuleName, LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ Protocol: *transportProto, FrontendIPConfiguration: &network.SubResource{ ID: to.StringPtr(lbFrontendIPConfigID), }, BackendAddressPool: &network.SubResource{ ID: to.StringPtr(lbBackendPoolID), }, LoadDistribution: loadDistribution, FrontendPort:     to.Int32Ptr(port.Port), BackendPort:      to.Int32Ptr(port.Port), EnableFloatingIP: to.BoolPtr(true), </ins> }, <del> LoadDistribution: loadDistribution, FrontendPort:     to.Int32Ptr(port.Port), BackendPort:      to.Int32Ptr(port.Port), EnableFloatingIP: to.BoolPtr(true), }, } if port.Protocol == v1.ProtocolTCP { expectedRule.LoadBalancingRulePropertiesFormat.IdleTimeoutInMinutes = lbIdleTimeout } </del> <ins> } if protocol == v1.ProtocolTCP { expectedRule.LoadBalancingRulePropertiesFormat.IdleTimeoutInMinutes = lbIdleTimeout } </ins> <del> // we didn't construct the probe objects for UDP or SCTP because they're not used/needed/allowed if port.Protocol != v1.ProtocolUDP && port.Protocol != v1.ProtocolSCTP { expectedRule.Probe = &network.SubResource{ ID: to.StringPtr(az.getLoadBalancerProbeID(lbName, lbRuleName)), </del> <ins> // we didn't construct the probe objects for UDP or SCTP because they're not used/needed/allowed if protocol != v1.ProtocolUDP && protocol != v1.ProtocolSCTP { expectedRule.Probe = &network.SubResource{ ID: to.StringPtr(az.getLoadBalancerProbeID(lbName, lbRuleName)), } </ins> } <del> } expectedRules = append(expectedRules, expectedRule) </del> <ins> expectedRules = append(expectedRules, expectedRule) } </ins> } return expectedProbes, expectedRules, nil

return clusterName } <del> func (az *Cloud) getLoadBalancerRuleName(service *v1.Service, port v1.ServicePort, subnetName *string) string { </del> <ins> func (az *Cloud) getLoadBalancerRuleName(service *v1.Service, protocol v1.Protocol, port int32, subnetName *string) string { </ins> prefix := az.getRulePrefix(service) if subnetName == nil { <del> return fmt.Sprintf("%s-%s-%d", prefix, port.Protocol, port.Port) </del> <ins> return fmt.Sprintf("%s-%s-%d", prefix, protocol, port) </ins> } <del> return fmt.Sprintf("%s-%s-%s-%d", prefix, *subnetName, port.Protocol, port.Port) </del> <ins> return fmt.Sprintf("%s-%s-%s-%d", prefix, *subnetName, protocol, port) </ins> } func (az *Cloud) getSecurityRuleName(service *v1.Service, port v1.ServicePort, sourceAddrPrefix string) string {

} for _, wantedRule := range svc.Spec.Ports { expectedRuleCount++ <del> wantedRuleName := az.getLoadBalancerRuleName(&svc, wantedRule, subnet(&svc)) </del> <ins> wantedRuleName := az.getLoadBalancerRuleName(&svc, wantedRule.Protocol, wantedRule.Port, subnet(&svc)) </ins> foundRule := false for _, actualRule := range *loadBalancer.LoadBalancingRules { if strings.EqualFold(*actualRule.Name, wantedRuleName) &&