Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Current azure cloud provider would fail to detach azure disk when there is server side error, need to add retry mechanism for detach disk operation, while for attach disk operation, it's not necessary since k8s pv-controller will try if first try failed. What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): v1.10 ~ master Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Others: /sig azure /assign
/milestone v1.14 We're entering and this looks relevant to the 1.14 release. Please if I am incorrect

Discussion:
}) } <ins> // UpdateVmssVMWithRetry invokes az.VirtualMachineScaleSetVMsClient.Update with exponential backoff retry func (az *Cloud) UpdateVmssVMWithRetry(resourceGroupName string, VMScaleSetName string, instanceID string, parameters compute.VirtualMachineScaleSetVM) error { return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { ctx, cancel := getContextWithCancel() defer cancel() resp, err := az.VirtualMachineScaleSetVMsClient.Update(ctx, resourceGroupName, VMScaleSetName, instanceID, parameters) klog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s,%s): end", VMScaleSetName, instanceID) return az.processHTTPRetryResponse(nil, "", resp, err) }) } </ins> // isSuccessHTTPResponse determines if the response from an HTTP request suggests success func isSuccessHTTPResponse(resp *http.Response) bool { if resp == nil {

// Invalidate the cache right after updating defer as.cloud.vmCache.Delete(vmName) <del> _, err = as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM) </del> <ins> resp, err := as.VirtualMachinesClient.CreateOrUpdate(ctx, nodeResourceGroup, vmName, newVM) if as.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { klog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, vmName, diskName, diskURI, err) retryErr := as.CreateOrUpdateVMWithRetry(nodeResourceGroup, vmName, newVM) if retryErr != nil { err = retryErr klog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, vmName, diskName, diskURI, err) } } </ins> if err != nil { <del> klog.Errorf("azureDisk - detach disk(%s) failed, err: %v", diskName, err) </del> <ins> klog.Errorf("azureDisk - detach disk(%s, %s)) failed, err: %v", diskName, diskURI, err) </ins> } else { <del> klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) </del> <ins> klog.V(2).Infof("azureDisk - detach disk(%s, %s) succeeded", diskName, diskURI) </ins> } return err }

key := buildVmssCacheKey(nodeResourceGroup, ss.makeVmssVMName(ssName, instanceID)) defer ss.vmssVMCache.Delete(key) <del> klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s)", nodeResourceGroup, nodeName, diskName) _, err = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM) </del> <ins> klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%s, %s)", nodeResourceGroup, nodeName, diskName, diskURI) resp, err := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM) if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) { klog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, nodeName, diskName, diskURI, err) retryErr := ss.UpdateVmssVMWithRetry(nodeResourceGroup, ssName, instanceID, newVM) if retryErr != nil { err = retryErr klog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s) detach disk(%s, %s), err: %v", nodeResourceGroup, nodeName, diskName, diskURI, err) } } </ins> if err != nil { <del> klog.Errorf("azureDisk - detach disk(%s) from %s failed, err: %v", diskName, nodeName, err) </del> <ins> klog.Errorf("azureDisk - detach disk(%s, %s) from %s failed, err: %v", diskName, diskURI, nodeName, err) </ins> } else { <del> klog.V(2).Infof("azureDisk - detach disk(%s) succeeded", diskName) </del> <ins> klog.V(2).Infof("azureDisk - detach disk(%s, %s) succeeded", diskName, diskURI) </ins> } return err