Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Recently we upgraded our cluster from 1.14.4 to 1.14.7. Then we started to observe tons of following errors: I guess the cause may be related to this line which was introduced recently: As we have hundreds of services with LoadBalancer type, and we also enable cluster-autoscaler, the nodes list change very often. For each service, there will be one call to GetScaleSet. The quota will be exhausted quickly. When fails, new nodes created by cluster-autoscaler will lose connectivity to load balancer. What you expected to happen: Kubernetes should not hit VMSS throttling threshold. How to reproduce it (as minimally and precisely as possible): a cluster with aks-engine. Use VMSS. 200 services with LoadBalancer type. up VMSS to 50 nodes down VMSS to 60 nodes scaling up and down VMSS Anything else we need to know?: Environment: Kubernetes version (use ): 1.14.7 Cloud provider or hardware configuration: Azure, VMSS OS (e.g: ): Ubuntu Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig cloud-provider /area provider/azure
I create a PR to mitigate the issue.
do we have a rough num about how this PR would mitigate this issue? looks like we need to cherry-pick this PR to old releases. it's better if we have a rough estimation.
The revised version of code calls instead of azure API. The cache refreshes itself every 10 min. I believe this will solve the throttling issue.
I believe we're being affected by this. Apart from getting the - when creating our to use a pre-configured Azure Public IP, the ingress appears to be getting a different IP each time
Thank you all!
The PR is merged. Does your issue remain?
pls cherry-pick that fix to old release, thanks
Cherry picked to release 1.14-1.16
I’ll retry today and let you know Wysłane z iPhone'a
Issue returned in K8S 1.15.4 Warning UpdateLoadBalancerFailed service/nginx-ingress-controller Error updating load balancer with new hosts map[<nodes names]: [EnsureHostInPool(ingress/nginx-ingress-controller): backendPoolID(/subscriptions/<subscription/resourceGroups/<rg/providers/Microsoft.Network/loadBalancers/<rg/backendAddressPools/<rg) - failed to ensure host in pool: "azure - cloud provider rate limited(read) for operation:VMSSGet", EnsureHostInPool(ingress/nginx-ingress-controller):

Discussion:
// (e.g. master nodes) may not belong to any scale sets. availabilitySet VMSet <ins> vmssCache                 *timedCache </ins> vmssVMCache               *timedCache availabilitySetNodesCache *timedCache }

return nil, err } <ins> ss.vmssCache, err = ss.newVMSSCache() if err != nil { return nil, err } </ins> ss.vmssVMCache, err = ss.newVMSSVirtualMachinesCache() if err != nil { return nil, err

return ss, nil } <ins> func (ss *scaleSet) getVMSS(vmssName string, crt cacheReadType) (*compute.VirtualMachineScaleSet, error) { getter := func(vmssName string) (*compute.VirtualMachineScaleSet, error) { cached, err := ss.vmssCache.Get(vmssKey, crt) if err != nil { return nil, err } vmsses := cached.(*sync.Map) if vmss, ok := vmsses.Load(vmssName); ok { result := vmss.(*vmssEntry) return result.vmss, nil } return nil, nil } vmss, err := getter(vmssName) if err != nil { return nil, err } if vmss != nil { return vmss, nil } klog.V(3).Infof("Couldn't find VMSS with name %s, refreshing the cache", vmssName) ss.vmssCache.Delete(vmssKey) vmss, err = getter(vmssName) if err != nil { return nil, err } if vmss == nil { return nil, cloudprovider.InstanceNotFound } return vmss, nil } </ins> // getVmssVM gets virtualMachineScaleSetVM by nodeName from cache. // It returns cloudprovider.InstanceNotFound if node does not belong to any scale sets. func (ss *scaleSet) getVmssVM(nodeName string, crt cacheReadType) (string, string, *compute.VirtualMachineScaleSetVM, error) {

} for vmssName := range vmssNamesMap { <del> vmss, err := ss.GetScaleSetWithRetry(service, ss.ResourceGroup, vmssName) </del> <ins> vmss, err := ss.getVMSS(vmssName, cacheReadTypeDefault) </ins> if err != nil { return err }

} for vmssName := range vmssNamesMap { <del> vmss, err := ss.GetScaleSetWithRetry(service, ss.ResourceGroup, vmssName) </del> <ins> vmss, err := ss.getVMSS(vmssName, cacheReadTypeDefault) </ins> // When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error. // Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.

package azure import ( <ins> "context" </ins> "strings" "sync" "time"

var ( vmssNameSeparator = "_" <ins> vmssKey                 = "k8svmssKey" </ins> vmssVirtualMachinesKey  = "k8svmssVirtualMachinesKey" availabilitySetNodesKey = "k8sAvailabilitySetNodesKey" availabilitySetNodesCacheTTL = 15 * time.Minute <ins> vmssTTL                      = 10 * time.Minute </ins> vmssVirtualMachinesTTL       = 10 * time.Minute )

lastUpdate     time.Time } <ins> type vmssEntry struct { vmss       *compute.VirtualMachineScaleSet lastUpdate time.Time } func (ss *scaleSet) newVMSSCache() (*timedCache, error) { getter := func(key string) (interface{}, error) { localCache := &sync.Map{} // [vmssName]*vmssEntry allResourceGroups, err := ss.GetResourceGroups() if err != nil { return nil, err } for _, resourceGroup := range allResourceGroups.List() { allScaleSets, err := ss.VirtualMachineScaleSetsClient.List(context.Background(), resourceGroup) if err != nil { klog.Errorf("VirtualMachineScaleSetsClient.List failed: %v", err) return nil, err } for _, scaleSet := range allScaleSets { if scaleSet.Name == nil || *scaleSet.Name == "" { klog.Warning("failed to get the name of VMSS") continue } localCache.Store(*scaleSet.Name, &vmssEntry{ vmss:       &scaleSet, lastUpdate: time.Now().UTC(), }) } } return localCache, nil } return newTimedcache(vmssTTL, getter) } </ins> func extractVmssVMName(name string) (string, string, error) { split := strings.SplitAfter(name, vmssNameSeparator) if len(split) < 2 {