Change:
<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind feature What happened: In certain large cluster scenarios we would like to anticipate kubelet behavior hitting request limits, and backoff in response. E.g., What you expected to happen: This is a request for add'l backoff coverage, so nothing surprising. How to reproduce it (as minimally and precisely as possible): Build a cluster with as many nodes as your subscription will support on azure until kubelet behavior contributes to backend API request limits. Environment: Kubernetes version (use ): and Cloud provider or hardware configuration**: azure OS (e.g. from /etc/os-release): Ubuntu 16.04.2 LTS (Xenial Xerus) Kernel (e.g. ): Linux k8s-pool7--7 4.4.0-83-generic -Ubuntu SMP Mon Jun 26 17:54:43 UTC 2017 x8664 x8664 x86_64 GNU/Linux Install tools: acs-engine Others:
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig azure

Discussion:
return machine, exists, err } <ins> // VirtualMachineClientGetWithRetry invokes az.VirtualMachinesClient.Get with exponential backoff retry func (az *Cloud) VirtualMachineClientGetWithRetry(resourceGroup, vmName string, types compute.InstanceViewTypes) (compute.VirtualMachine, error) { var machine compute.VirtualMachine err := wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) { var retryErr error machine, retryErr = az.VirtualMachinesClient.Get(resourceGroup, vmName, types) if retryErr != nil { glog.Errorf("backoff: failure, will retry,err=%v", retryErr) return false, nil } glog.V(2).Infof("backoff: success") return true, nil }) return machine, err } // GetIPForMachineWithRetry invokes az.getIPForMachine with exponential backoff retry func (az *Cloud) GetIPForMachineWithRetry(name types.NodeName) (string, error) { var ip string err := wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) { var retryErr error ip, retryErr = az.getIPForMachine(name) if retryErr != nil { glog.Errorf("backoff: failure, will retry,err=%v", retryErr) return false, nil } glog.V(2).Infof("backoff: success") return true, nil }) return ip, err } </ins> // CreateOrUpdateSGWithRetry invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry func (az *Cloud) CreateOrUpdateSGWithRetry(sg network.SecurityGroup) error { return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {

} ip, err := az.getIPForMachine(name) if err != nil { <del> glog.Errorf("error: az.NodeAddresses, az.getIPForMachine(%s), err=%v", name, err) return nil, err </del> <ins> if az.CloudProviderBackoff { glog.V(2).Infof("NodeAddresses(%s) backing off", name) ip, err = az.GetIPForMachineWithRetry(name) if err != nil { glog.V(2).Infof("NodeAddresses(%s) abort backoff", name) return nil, err } } else { glog.Errorf("error: az.NodeAddresses, az.getIPForMachine(%s), err=%v", name, err) return nil, err } </ins> } return []v1.NodeAddress{

utilerrors "k8s.io/apimachinery/pkg/util/errors" serviceapi "k8s.io/kubernetes/pkg/api/v1/service" <ins> "github.com/Azure/azure-sdk-for-go/arm/compute" </ins> "github.com/Azure/azure-sdk-for-go/arm/network" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/glog"

// This ensures the given VM's Primary NIC's Primary IP Configuration is // participating in the specified LoadBalancer Backend Pool. func (az *Cloud) ensureHostInPool(serviceName string, nodeName types.NodeName, backendPoolID string) error { <ins> var machine compute.VirtualMachine </ins> vmName := mapNodeNameToVMName(nodeName) az.operationPollRateLimiter.Accept() machine, err := az.VirtualMachinesClient.Get(az.ResourceGroup, vmName, "") if err != nil { <del> return err </del> <ins> if az.CloudProviderBackoff { glog.V(2).Infof("ensureHostInPool(%s, %s, %s) backing off", serviceName, nodeName, backendPoolID) machine, err = az.VirtualMachineClientGetWithRetry(az.ResourceGroup, vmName, "") if err != nil { glog.V(2).Infof("ensureHostInPool(%s, %s, %s) abort backoff", serviceName, nodeName, backendPoolID) return err } } else { return err } </ins> } primaryNicID, err := getPrimaryInterfaceID(machine)

} func (az *Cloud) getIPForMachine(nodeName types.NodeName) (string, error) { <ins> az.operationPollRateLimiter.Accept() </ins> machine, exists, err := az.getVirtualMachine(nodeName) if !exists { return "", cloudprovider.InstanceNotFound