Change:
What happened: We saw several occurrences of kube-controller-manager failing to check a k8s node existence after we deleted that node's parent VMSS (and by extension, cloud instance). The leaked k8s nodes remained present in the cluster until manually deleted. Cloud provider retries caused a VMSS VMs API call surge, degenerating in hard throttling for the whole subscription. In those situations, kube-controller-manager would continuously log errors like the following: Looking at Azure cloud-provider implementation shows that all the VMSS API calls errors (including a 404/not found on the VMSS object) are bubbled up as generic errors, and never used as an indication of a missing instance. What you expected to happen: k8s nodes being removed as their VMSS went missing. How to reproduce it (as minimally and precisely as possible): Create and delete many VMSS providing instances registering to a cluster. Environment: Kubernetes version (use ): 1.19.0 Cloud provider or hardware configuration: Azure (self managed cluster, non AKS, ) OS (e.g: ): Linux, Ubuntu Focal 20.04.1 LTS Kernel (e.g. ): 5.4.0-1022-azure /sig cloud-provider /area provider/azure
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
"k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient" "k8s.io/legacy-cloud-providers/azure/clients/publicipclient/mockpublicipclient" "k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient" <ins> "k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient" "k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/mockvmssvmclient" </ins> "k8s.io/legacy-cloud-providers/azure/retry" )

assert.Equal(t, test.expectedErrMsg, err, test.name) assert.Equal(t, test.expected, exist, test.name) } <ins> vmssTestCases := []struct { name       string providerID string scaleSet   string vmList     []string expected   bool rerr       *retry.Error }{ { name:       "InstanceExistsByProviderID should return true if VMSS and VM exist", providerID: "azure:///subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmssee6c2/virtualMachines/0", scaleSet:   "vmssee6c2", vmList:     []string{"vmssee6c2000000"}, expected:   true, }, { name:       "InstanceExistsByProviderID should return false if VMSS exist but VM doesn't", providerID: "azure:///subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmssee6c2/virtualMachines/0", scaleSet:   "vmssee6c2", expected:   false, }, { name:       "InstanceExistsByProviderID should return false if VMSS doesn't exist", providerID: "azure:///subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/missing-vmss/virtualMachines/0", rerr:       &retry.Error{HTTPStatusCode: 404}, expected:   false, }, } for _, test := range vmssTestCases { ss, err := newTestScaleSet(ctrl) assert.NoError(t, err, test.name) cloud.VMSet = ss mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) mockVMSSVMClient := mockvmssvmclient.NewMockInterface(ctrl) ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient expectedScaleSet := buildTestVMSS(test.scaleSet, test.scaleSet) mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, test.rerr).AnyTimes() expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "succeeded", false) mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, test.rerr).AnyTimes() mockVMsClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) mockVMsClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachine{}, nil).AnyTimes() exist, _ := cloud.InstanceExistsByProviderID(context.Background(), test.providerID) assert.Equal(t, test.expected, exist, test.name) } </ins> } func TestNodeAddressesByProviderID(t *testing.T) {

allVMs, rerr := ss.VirtualMachineScaleSetVMsClient.List(ctx, resourceGroup, scaleSetName, string(compute.InstanceView)) if rerr != nil { klog.Errorf("VirtualMachineScaleSetVMsClient.List failed: %v", rerr) <ins> if rerr.IsNotFound() { return nil, cloudprovider.InstanceNotFound } </ins> return nil, rerr.Error() }

return err.HTTPStatusCode == http.StatusTooManyRequests || err.RetryAfter.After(now()) } <ins> // IsNotFound returns true the if the requested object wasn't found func (err *Error) IsNotFound() bool { if err == nil { return false } return err.HTTPStatusCode == http.StatusNotFound } </ins> // NewError creates a new Error. func NewError(retriable bool, err error) *Error { return &Error{