Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: controller-manager is stuck in deleting an incorrectly configured LB type service. it cannot be recovered until restarted. What you expected to happen: controller-manager should recover. At least it should allow us to create the service without restarting it. How to reproduce it (as minimally and precisely as possible): put a random IP in above yaml and make sure the resource group doen't exist. Anything else we need to know?: Environment: Kubernetes version (use ): 1.12.4 Cloud provider or hardware configuration: AKS OS (e.g: ): Kernel (e.g. ): Install tools: Others:
/sig azure
I also want to add that the above repro is stuck in 403. It's possible for it to be stuck in 404 as well when the SP has the right permission.
Can you share the log for 404?
Thanks for reporting the issue. /assign Do you mean other services couldn't be configured when this happens?
correct.
FYI, we had the same issue when the Idle Timeout annotation was set to "3" which isn't a supported case. Will this PR also cover this edge case ?
Unfortunately , this is still not fixed. The idle timeout is not 403 or 404. We need another fix for this (and possibly more for other annotations)

Discussion:
func (az *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error { isInternal := requiresInternalLoadBalancer(service) serviceName := getServiceName(service) <del> klog.V(5).Infof("delete(%s): START clusterName=%q", serviceName, clusterName) </del> <ins> klog.V(5).Infof("Delete service (%s): START clusterName=%q", serviceName, clusterName) ignoreErrors := func(err error) error { if ignoreStatusNotFoundFromError(err) == nil { klog.V(5).Infof("EnsureLoadBalancerDeleted: ignoring StatusNotFound error because the resource doesn't exist (%v)", err) return nil } if ignoreStatusForbiddenFromError(err) == nil { klog.V(5).Infof("EnsureLoadBalancerDeleted: ignoring StatusForbidden error (%v). This may be caused by wrong configuration via service annotations", err) return nil } return err } </ins> serviceIPToCleanup, err := az.findServiceIPAddress(ctx, clusterName, service, isInternal) <del> if err != nil { </del> <ins> if ignoreErrors(err) != nil { </ins> return err } klog.V(2).Infof("EnsureLoadBalancerDeleted: reconciling security group for service %q with IP %q, wantLb = false", serviceName, serviceIPToCleanup) if _, err := az.reconcileSecurityGroup(clusterName, service, &serviceIPToCleanup, false /* wantLb */); err != nil { <del> return err </del> <ins> if ignoreErrors(err) != nil { return err } </ins> } if _, err := az.reconcileLoadBalancer(clusterName, service, nil, false /* wantLb */); err != nil { <del> return err </del> <ins> if ignoreErrors(err) != nil { return err } </ins> } if _, err := az.reconcilePublicIP(clusterName, service, nil, false /* wantLb */); err != nil { <del> return err </del> <ins> if ignoreErrors(err) != nil { return err } </ins> } <del> klog.V(2).Infof("delete(%s): FINISH", serviceName) </del> <ins> klog.V(2).Infof("Delete service (%s): FINISH", serviceName) </ins> return nil }

package azure import ( <ins> "context" </ins> "fmt" "reflect" "testing"

assert.Equal(t, c.expected, real, fmt.Sprintf("TestCase[%d]: %s", i, c.desc)) } } <ins> func TestEnsureLoadBalancerDeleted(t *testing.T) { const vmCount = 8 const availabilitySetCount = 4 const serviceCount = 9 tests := []struct { desc              string service           v1.Service expectCreateError bool }{ { desc:    "external service should be created and deleted successfully", service: getTestService("test1", v1.ProtocolTCP, 80), }, { desc:    "internal service should be created and deleted successfully", service: getInternalTestService("test2", 80), }, { desc:    "annotated service with same resourceGroup should be created and deleted successfully", service: getResourceGroupTestService("test3", "rg", "", 80), }, { desc:              "annotated service with different resourceGroup shouldn't be created but should be deleted successfully", service:           getResourceGroupTestService("test4", "random-rg", "1.2.3.4", 80), expectCreateError: true, }, } az := getTestCloud() for i, c := range tests { clusterResources := getClusterResources(az, vmCount, availabilitySetCount) getTestSecurityGroup(az) if c.service.Annotations[ServiceAnnotationLoadBalancerInternal] == "true" { addTestSubnet(t, az, &c.service) } // create the service first. lbStatus, err := az.EnsureLoadBalancer(context.TODO(), testClusterName, &c.service, clusterResources.nodes) if c.expectCreateError { assert.NotNil(t, err, "TestCase[%d]: %s", i, c.desc) } else { assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) assert.NotNil(t, lbStatus, "TestCase[%d]: %s", i, c.desc) result, err := az.LoadBalancerClient.List(context.TODO(), az.Config.ResourceGroup) assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) assert.Equal(t, len(result), 1, "TestCase[%d]: %s", i, c.desc) assert.Equal(t, len(*result[0].LoadBalancingRules), 1, "TestCase[%d]: %s", i, c.desc) } // finally, delete it. err = az.EnsureLoadBalancerDeleted(context.TODO(), testClusterName, &c.service) assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) result, err := az.LoadBalancerClient.List(context.Background(), az.Config.ResourceGroup) assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) assert.Equal(t, len(result), 0, "TestCase[%d]: %s", i, c.desc) } } </ins>

return svc } <ins> func getResourceGroupTestService(identifier, resourceGroup, loadBalancerIP string, requestedPorts ...int32) v1.Service { svc := getTestService(identifier, v1.ProtocolTCP, requestedPorts...) svc.Spec.LoadBalancerIP = loadBalancerIP svc.Annotations[ServiceAnnotationLoadBalancerResourceGroup] = resourceGroup return svc } </ins> func setLoadBalancerModeAnnotation(service *v1.Service, lbMode string) { service.Annotations[ServiceAnnotationLoadBalancerMode] = lbMode }

return err } <ins> // ignoreStatusForbiddenFromError returns nil if the status code is StatusForbidden. // This happens when AuthorizationFailed is reported from Azure API. func ignoreStatusForbiddenFromError(err error) error { if err == nil { return nil } v, ok := err.(autorest.DetailedError) if ok && v.StatusCode == http.StatusForbidden { return nil } return err } </ins> /// getVirtualMachine calls 'VirtualMachinesClient.Get' with a timed cache /// The service side has throttling control that delays responses if there're multiple requests onto certain vm /// resource request in short period.