Repository: kubernetes-sigs/cloud-provider-azure
Pull request #2024 (2022-07-14 04:14:32)

[release 1.0] Add detailed error reuslt_code in ObserveOperationWithR…
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?
/kind feature
<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

#### What this PR does / why we need it:
When errors happen, the current result_code is always "failed" in ObserveOperationWithResult func.
Now the result_code turns to something like "failed_create_route",  "failed_ensure_loadbalancer" and so on.
#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #1986 

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
Result_code in latency logs is more detailed.
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```



Patch:
diff --git a/go.mod b/go.mod
index 4e9587415..5abc65e06 100644
--- a/go.mod
+++ b/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/rubiojr/go-vhd v0.0.0-20200706122120-ccecf6c0760f
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.8.0
+	github.com/stretchr/testify v1.7.5
 	golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
 	golang.org/x/text v0.3.7
 	k8s.io/api v0.21.11
@@ -27,7 +27,7 @@ require (
 	k8s.io/component-base v0.21.11
 	k8s.io/controller-manager v0.21.11
 	k8s.io/klog/v2 v2.9.0
-	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
+	k8s.io/utils v0.0.0-20211116205334-6203023598ed
 	sigs.k8s.io/yaml v1.2.0
 )
 
diff --git a/go.sum b/go.sum
index b3bc67942..29f6aac33 100644
--- a/go.sum
+++ b/go.sum
@@ -453,8 +453,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q=
+github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
@@ -858,9 +858,8 @@ k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20211110012726-3cc51fd1e909 h1:s77MRc/+/eQjsF89MB12JssAlsoi9mnNoaacRqibeAU=
 k8s.io/kube-openapi v0.0.0-20211110012726-3cc51fd1e909/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
+k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=
 k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc=
-k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/pkg/azureclients/armclient/azure_armclient.go b/pkg/azureclients/armclient/azure_armclient.go
index ff136c083..be24d7221 100644
--- a/pkg/azureclients/armclient/azure_armclient.go
+++ b/pkg/azureclients/armclient/azure_armclient.go
@@ -126,7 +126,7 @@ func NormalizeAzureRegion(name string) string {
 	return strings.ToLower(region)
 }
 
-// DoHackRegionalRetryDecorator returns an autorest.SendDecorator which performs retry with customizable backoff policy.
+// DoExponentialBackoffRetry returns an autorest.SendDecorator which performs retry with customizable backoff policy.
 func DoHackRegionalRetryDecorator(c *Client) autorest.SendDecorator {
 	return func(s autorest.Sender) autorest.Sender {
 		return autorest.SenderFunc(func(request *http.Request) (*http.Response, error) {
@@ -135,35 +135,28 @@ func DoHackRegionalRetryDecorator(c *Client) autorest.SendDecorator {
 				klog.V(2).Infof("response is empty")
 				return response, rerr
 			}
-
+			if rerr == nil || response.StatusCode == http.StatusNotFound || c.regionalEndpoint == "" {
+				return response, rerr
+			}
+			// Hack: retry the regional ARM endpoint in case of ARM traffic split and arm resource group replication is too slow
 			bodyBytes, _ := ioutil.ReadAll(response.Body)
 			defer func() {
 				response.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
 			}()
 
 			bodyString := string(bodyBytes)
-			trimmed := strings.TrimSpace(bodyString)
-			// Hack: retry the regional ARM endpoint in case of ARM traffic split and arm resource group replication is too slow
-			// Empty content and 2xx http status code are returned in this case.
-			// Issue: https://github.com/kubernetes-sigs/cloud-provider-azure/issues/1296
-			emptyResp := (response.ContentLength == 0 || trimmed == "" || trimmed == "{}") && response.StatusCode >= 200 && response.StatusCode < 300
-			if !emptyResp {
-				if rerr == nil || response.StatusCode == http.StatusNotFound || c.regionalEndpoint == "" {
-					return response, rerr
-				}
-
-				var body map[string]interface{}
-				if e := json.Unmarshal(bodyBytes, &body); e != nil {
-					klog.Errorf("Send.sendRequest: error in parsing response body string: %s, Skip retrying regional host", e.Error())
-					return response, rerr
-				}
-				klog.V(5).Infof("Send.sendRequest original response: %s", bodyString)
+			var body map[string]interface{}
+			if e := json.Unmarshal(bodyBytes, &body); e != nil {
+				klog.Errorf("Send.sendRequest: error in parsing response body string: %s, Skip retrying regional host", e.Error())
+				return response, rerr
+			}
+			klog.V(5).Infof("Send.sendRequest original response: %s", bodyString)
 
-				err, ok := body["error"].(map[string]interface{})
-				if !ok || err["code"] == nil || !strings.EqualFold(err["code"].(string), "ResourceGroupNotFound") {
-					klog.V(5).Infof("Send.sendRequest: response body does not contain ResourceGroupNotFound error code. Skip retrying regional host")
-					return response, rerr
-				}
+			if err, ok := body["error"].(map[string]interface{}); !ok ||
+				err["code"] == nil ||
+				!strings.EqualFold(err["code"].(string), "ResourceGroupNotFound") {
+				klog.V(5).Infof("Send.sendRequest: response body does not contain ResourceGroupNotFound error code. Skip retrying regional host")
+				return response, rerr
 			}
 
 			currentHost := request.URL.Host
diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go
index 19aab2480..90b9b27ad 100644
--- a/pkg/consts/consts.go
+++ b/pkg/consts/consts.go
@@ -277,10 +277,6 @@ const (
 	// is `a=b,c=d,...`. After updated, the old user-assigned tags would not be replaced by the new ones.
 	ServiceAnnotationAzurePIPTags = "service.beta.kubernetes.io/azure-pip-tags"
 
-	// ServiceAnnotationDisableLoadBalancerFloatingIP is the annotation used on the service to disable floating IP in load balancer rule.
-	// If omitted, the default value is false
-	ServiceAnnotationDisableLoadBalancerFloatingIP = "service.beta.kubernetes.io/azure-disable-load-balancer-floating-ip"
-
 	// ServiceAnnotationAzurePIPTags sets the additional Public IPs (split by comma) besides the service's Public IP configured on LoadBalancer.
 	// These additional Public IPs would be consumed by kube-proxy to configure the iptables rules on each node. Note they would not be configured
 	// automatically on Azure LoadBalancer. Instead, they need to be configured manually (e.g. on Azure cross-region LoadBalancer by another operator).
diff --git a/pkg/consts/helpers.go b/pkg/consts/helpers.go
index 13f49dd44..44296f3d9 100644
--- a/pkg/consts/helpers.go
+++ b/pkg/consts/helpers.go
@@ -40,11 +40,6 @@ func IsK8sServiceInternalIPv6(service *v1.Service) bool {
 	return IsK8sServiceUsingInternalLoadBalancer(service) && net.IsIPv6String(service.Spec.ClusterIP)
 }
 
-// IsK8sServiceDisableLoadBalancerFloatingIP return if floating IP in load balancer is disabled in kubernetes service annotations
-func IsK8sServiceDisableLoadBalancerFloatingIP(service *v1.Service) bool {
-	return expectAttributeInSvcAnnotationBeEqualTo(service.Annotations, ServiceAnnotationDisableLoadBalancerFloatingIP, TrueAnnotationValue)
-}
-
 // GetHealthProbeConfigOfPortFromK8sSvcAnnotation get health probe configuration for port
 func GetHealthProbeConfigOfPortFromK8sSvcAnnotation(annotations map[string]string, port int32, key HealthProbeParams, validators ...BusinessValidator) (*string, error) {
 	return GetAttributeValueInSvcAnnotation(annotations, BuildHealthProbeAnnotationKeyForPort(port, key), validators...)
diff --git a/pkg/metrics/azure_metrics.go b/pkg/metrics/azure_metrics.go
index be2a7ae0c..70acb3066 100644
--- a/pkg/metrics/azure_metrics.go
+++ b/pkg/metrics/azure_metrics.go
@@ -99,6 +99,9 @@ func (mc *MetricContext) ObserveOperationWithResult(isOperationSucceeded bool, l
 	resultCode := "succeeded"
 	if !isOperationSucceeded {
 		resultCode = "failed"
+		if len(mc.attributes) > 0 {
+			resultCode += mc.attributes[0][strings.Index(mc.attributes[0], "_"):]
+		}
 		mc.CountFailedOperation()
 	}
 	mc.logLatency(3, latency, append(labelAndValues, "result_code", resultCode)...)
diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go
index 18203ffa9..10c90cf44 100644
--- a/pkg/provider/azure_loadbalancer.go
+++ b/pkg/provider/azure_loadbalancer.go
@@ -105,10 +105,8 @@ func (az *Cloud) reconcileService(ctx context.Context, clusterName string, servi
 	if lbStatus != nil && len(lbStatus.Ingress) > 0 {
 		serviceIP = &lbStatus.Ingress[0].IP
 	}
-
-	backendPrivateIPs := az.GetBackendPrivateIPs(clusterName, service, lb)
 	klog.V(2).Infof("reconcileService: reconciling security group for service %q with IP %q, wantLb = true", serviceName, logSafe(serviceIP))
-	if _, err := az.reconcileSecurityGroup(clusterName, service, serviceIP, &backendPrivateIPs, true /* wantLb */); err != nil {
+	if _, err := az.reconcileSecurityGroup(clusterName, service, serviceIP, true /* wantLb */); err != nil {
 		klog.Errorf("reconcileSecurityGroup(%s) failed: %#v", serviceName, err)
 		return nil, err
 	}
@@ -137,43 +135,6 @@ func (az *Cloud) reconcileService(ctx context.Context, clusterName string, servi
 	return lbStatus, nil
 }
 
-func (az *Cloud) GetBackendPrivateIPs(clusterName string, service *v1.Service, lb *network.LoadBalancer) []string {
-	serviceName := getServiceName(service)
-	lbBackendPoolName := getBackendPoolName(clusterName, service)
-	if lb.LoadBalancerPropertiesFormat == nil || lb.LoadBalancerPropertiesFormat.BackendAddressPools == nil {
-		return nil
-	}
-
-	backendPrivateIPs := sets.NewString()
-	for _, bp := range *lb.BackendAddressPools {
-		if strings.EqualFold(to.String(bp.Name), lbBackendPoolName) {
-			klog.V(10).Infof("bc.GetBackendPrivateIPs for service (%s): found wanted backendpool %s", serviceName, to.String(bp.Name))
-			if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil {
-				for _, backendIPConfig := range *bp.BackendIPConfigurations {
-					ipConfigID := to.String(backendIPConfig.ID)
-					nodeName, _, err := az.VMSet.GetNodeNameByIPConfigurationID(ipConfigID)
-					if err != nil {
-						klog.Errorf("bc.GetBackendPrivateIPs for service (%s): GetNodeNameByIPConfigurationID failed with error: %v", serviceName, err)
-						continue
-					}
-					privateIPs, err := az.VMSet.GetPrivateIPsByNodeName(nodeName)
-					if err != nil {
-						klog.Errorf("bc.GetBackendPrivateIPs for service (%s): GetPrivateIPsByNodeName(%s) failed with error: %v", serviceName, nodeName, err)
-						continue
-					}
-					if privateIPs != nil {
-						klog.V(2).Infof("bc.GetBackendPrivateIPs for service (%s): lb backendpool - found private IPs %v of node %s", serviceName, privateIPs, nodeName)
-						backendPrivateIPs.Insert(privateIPs...)
-					}
-				}
-			}
-		} else {
-			klog.V(10).Infof("bc.GetBackendPrivateIPs for service (%s): found unmanaged backendpool %s", serviceName, to.String(bp.Name))
-		}
-	}
-	return backendPrivateIPs.List()
-}
-
 // EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer
 func (az *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
 	// When a client updates the internal load balancer annotation,
@@ -251,7 +212,7 @@ func (az *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName stri
 	}
 
 	klog.V(2).Infof("EnsureLoadBalancerDeleted: reconciling security group for service %q with IP %q, wantLb = false", serviceName, serviceIPToCleanup)
-	_, err = az.reconcileSecurityGroup(clusterName, service, &serviceIPToCleanup, &[]string{}, false /* wantLb */)
+	_, err = az.reconcileSecurityGroup(clusterName, service, &serviceIPToCleanup, false /* wantLb */)
 	if err != nil {
 		return err
 	}
@@ -1374,7 +1335,10 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend
 				return true, nil
 			}
 		}
-		return loadBalancerIP != "" && !strings.EqualFold(loadBalancerIP, to.String(config.PrivateIPAddress)), nil
+		if loadBalancerIP == "" {
+			return config.PrivateIPAllocationMethod == network.IPAllocationMethodStatic, nil
+		}
+		return config.PrivateIPAllocationMethod != network.IPAllocationMethodStatic || !strings.EqualFold(loadBalancerIP, to.String(config.PrivateIPAddress)), nil
 	}
 	pipName, _, err := az.determinePublicIPName(clusterName, service, pips)
 	if err != nil {
@@ -2382,10 +2346,6 @@ func (az *Cloud) getExpectedLBRules(
 					ID: to.StringPtr(az.getLoadBalancerProbeID(lbName, az.getLoadBalancerResourceGroup(), *nodeEndpointHealthprobe.Name)),
 				}
 			}
-			if consts.IsK8sServiceDisableLoadBalancerFloatingIP(service) {
-				props.BackendPort = to.Int32Ptr(port.NodePort)
-				props.EnableFloatingIP = to.BoolPtr(false)
-			}
 			expectedRules = append(expectedRules, network.LoadBalancingRule{
 				Name:                              &lbRuleName,
 				LoadBalancingRulePropertiesFormat: props,
@@ -2468,7 +2428,7 @@ func (az *Cloud) getExpectedHAModeLoadBalancingRuleProperties(
 
 // This reconciles the Network Security Group similar to how the LB is reconciled.
 // This entails adding required, missing SecurityRules and removing stale rules.
-func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, lbIP *string, backendIPAddresses *[]string, wantLb bool) (*network.SecurityGroup, error) {
+func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service, lbIP *string, wantLb bool) (*network.SecurityGroup, error) {
 	serviceName := getServiceName(service)
 	klog.V(5).Infof("reconcileSecurityGroup(%s): START clusterName=%q", serviceName, clusterName)
 
@@ -2486,11 +2446,6 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service,
 		return nil, err
 	}
 
-	disableFloatingIP := false
-	if consts.IsK8sServiceDisableLoadBalancerFloatingIP(service) {
-		disableFloatingIP = true
-	}
-
 	destinationIPAddress := ""
 	if wantLb && lbIP == nil {
 		return nil, fmt.Errorf("no load balancer IP for setting up security rules for service %s", service.Name)
@@ -2534,7 +2489,7 @@ func (az *Cloud) reconcileSecurityGroup(clusterName string, service *v1.Service,
 		sourceAddressPrefixes = append(sourceAddressPrefixes, serviceTags...)
 	}
 
-	expectedSecurityRules, err := az.getExpectedSecurityRules(wantLb, ports, sourceAddressPrefixes, service, destinationIPAddresses, sourceRanges, *backendIPAddresses, disableFloatingIP)
+	expectedSecurityRules, err := az.getExpectedSecurityRules(wantLb, ports, sourceAddressPrefixes, service, destinationIPAddresses, sourceRanges)
 	if err != nil {
 		return nil, err
 	}
@@ -2668,15 +2623,13 @@ func (az *Cloud) reconcileSecurityRules(sg network.SecurityGroup, service *v1.Se
 		}
 	}
 
-	updatedRules = removeDuplicatedSecurityRules(updatedRules)
-
 	for _, r := range updatedRules {
 		klog.V(10).Infof("Updated security rule while processing %s: %s:%s -> %s:%s", service.Name, logSafe(r.SourceAddressPrefix), logSafe(r.SourcePortRange), logSafeCollection(r.DestinationAddressPrefix, r.DestinationAddressPrefixes), logSafe(r.DestinationPortRange))
 	}
 	return dirtySg, updatedRules, nil
 }
 
-func (az *Cloud) getExpectedSecurityRules(wantLb bool, ports []v1.ServicePort, sourceAddressPrefixes []string, service *v1.Service, destinationIPAddresses []string, sourceRanges utilnet.IPNetSet, backendIPAddresses []string, disableFloatingIP bool) ([]network.SecurityRule, error) {
+func (az *Cloud) getExpectedSecurityRules(wantLb bool, ports []v1.ServicePort, sourceAddressPrefixes []string, service *v1.Service, destinationIPAddresses []string, sourceRanges utilnet.IPNetSet) ([]network.SecurityRule, error) {
 	expectedSecurityRules := []network.SecurityRule{}
 
 	if wantLb {
@@ -2687,10 +2640,6 @@ func (az *Cloud) getExpectedSecurityRules(wantLb bool, ports []v1.ServicePort, s
 			if err != nil {
 				return nil, err
 			}
-			dstPort := port.Port
-			if disableFloatingIP {
-				dstPort = port.NodePort
-			}
 			for j := range sourceAddressPrefixes {
 				ix := i*len(sourceAddressPrefixes) + j
 				securityRuleName := az.getSecurityRuleName(service, port, sourceAddressPrefixes[j])
@@ -2699,16 +2648,13 @@ func (az *Cloud) getExpectedSecurityRules(wantLb bool, ports []v1.ServicePort, s
 					SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
 						Protocol:             *securityProto,
 						SourcePortRange:      to.StringPtr("*"),
-						DestinationPortRange: to.StringPtr(strconv.Itoa(int(dstPort))),
+						DestinationPortRange: to.StringPtr(strconv.Itoa(int(port.Port))),
 						SourceAddressPrefix:  to.StringPtr(sourceAddressPrefixes[j]),
 						Access:               network.SecurityRuleAccessAllow,
 						Direction:            network.SecurityRuleDirectionInbound,
 					},
 				}
-
-				if len(destinationIPAddresses) == 1 && disableFloatingIP {
-					nsgRule.DestinationAddressPrefixes = to.StringSlicePtr(backendIPAddresses)
-				} else if len(destinationIPAddresses) == 1 && !disableFloatingIP {
+				if len(destinationIPAddresses) == 1 {
 					// continue to use DestinationAddressPrefix to avoid NSG updates for existing rules.
 					nsgRule.DestinationAddressPrefix = to.StringPtr(destinationIPAddresses[0])
 				} else {
@@ -3252,12 +3198,12 @@ func equalLoadBalancingRulePropertiesFormat(s *network.LoadBalancingRuleProperti
 		properties = properties && reflect.DeepEqual(to.Bool(s.EnableTCPReset), to.Bool(t.EnableTCPReset))
 	}
 
-	properties = properties && equalSubResource(s.FrontendIPConfiguration, t.FrontendIPConfiguration) &&
-		equalSubResource(s.BackendAddressPool, t.BackendAddressPool) &&
+	properties = properties && reflect.DeepEqual(s.FrontendIPConfiguration, t.FrontendIPConfiguration) &&
+		reflect.DeepEqual(s.BackendAddressPool, t.BackendAddressPool) &&
 		reflect.DeepEqual(s.LoadDistribution, t.LoadDistribution) &&
 		reflect.DeepEqual(s.FrontendPort, t.FrontendPort) &&
 		reflect.DeepEqual(s.BackendPort, t.BackendPort) &&
-		equalSubResource(s.Probe, t.Probe) &&
+		reflect.DeepEqual(s.Probe, t.Probe) &&
 		reflect.DeepEqual(s.EnableFloatingIP, t.EnableFloatingIP) &&
 		reflect.DeepEqual(to.Bool(s.DisableOutboundSnat), to.Bool(t.DisableOutboundSnat))
 
@@ -3267,16 +3213,6 @@ func equalLoadBalancingRulePropertiesFormat(s *network.LoadBalancingRuleProperti
 	return properties
 }
 
-func equalSubResource(s *network.SubResource, t *network.SubResource) bool {
-	if s == nil && t == nil {
-		return true
-	}
-	if s == nil || t == nil {
-		return false
-	}
-	return strings.EqualFold(to.String(s.ID), to.String(t.ID))
-}
-
 // This compares rule's Name, Protocol, SourcePortRange, DestinationPortRange, SourceAddressPrefix, Access, and Direction.
 // Note that it compares rule's DestinationAddressPrefix only when it's not consolidated rule as such rule does not have DestinationAddressPrefix defined.
 // We intentionally do not compare DestinationAddressPrefixes in consolidated case because reconcileSecurityRule has to consider the two rules equal,
diff --git a/pkg/provider/azure_utils.go b/pkg/provider/azure_utils.go
index 4a0e018f2..57358fc31 100644
--- a/pkg/provider/azure_utils.go
+++ b/pkg/provider/azure_utils.go
@@ -274,15 +274,3 @@ func SliceEqual(s1, s2 []string) bool {
 	}
 	return true
 }
-
-func removeDuplicatedSecurityRules(rules []network.SecurityRule) []network.SecurityRule {
-	ruleNames := make(map[string]bool)
-	for i := len(rules) - 1; i >= 0; i-- {
-		if _, ok := ruleNames[to.String(rules[i].Name)]; ok {
-			klog.Warningf("Found duplicated rule %s, will be removed.", to.String(rules[i].Name))
-			rules = append(rules[:i], rules[i+1:]...)
-		}
-		ruleNames[to.String(rules[i].Name)] = true
-	}
-	return rules
-}
diff --git a/pkg/provider/azure_vmss_cache.go b/pkg/provider/azure_vmss_cache.go
index 8cb141d61..acdf7ab31 100644
--- a/pkg/provider/azure_vmss_cache.go
+++ b/pkg/provider/azure_vmss_cache.go
@@ -62,15 +62,9 @@ func (ss *ScaleSet) newVMSSCache() (*azcache.TimedCache, error) {
 			return nil, err
 		}
 
-		resourceGroupNotFound := false
 		for _, resourceGroup := range allResourceGroups.List() {
 			allScaleSets, rerr := ss.VirtualMachineScaleSetsClient.List(context.Background(), resourceGroup)
 			if rerr != nil {
-				if rerr.IsNotFound() {
-					klog.Warningf("Skip caching vmss for resource group %s due to error: %v", resourceGroup, rerr.Error())
-					resourceGroupNotFound = true
-					continue
-				}
 				klog.Errorf("VirtualMachineScaleSetsClient.List failed: %v", rerr)
 				return nil, rerr.Error()
 			}
@@ -89,23 +83,6 @@ func (ss *ScaleSet) newVMSSCache() (*azcache.TimedCache, error) {
 			}
 		}
 
-		if resourceGroupNotFound {
-			// gc vmss vm cache when there is resource group not found
-			removed := map[string]bool{}
-			ss.vmssVMCache.Range(func(key, value interface{}) bool {
-				cacheKey := key.(string)
-				vlistIdx := cacheKey[strings.LastIndex(cacheKey, "/")+1:]
-				if _, ok := localCache.Load(vlistIdx); !ok {
-					klog.V(2).Infof("remove vmss %s from cache due to rg not found", cacheKey)
-					removed[cacheKey] = true
-				}
-				return true
-			})
-
-			for key := range removed {
-				ss.vmssVMCache.Delete(key)
-			}
-		}
 		return localCache, nil
 	}
 
diff --git a/vendor/k8s.io/utils/pointer/pointer.go b/vendor/k8s.io/utils/pointer/pointer.go
index f5802d2e8..2cab2c580 100644
--- a/vendor/k8s.io/utils/pointer/pointer.go
+++ b/vendor/k8s.io/utils/pointer/pointer.go
@@ -19,7 +19,6 @@ package pointer
 import (
 	"fmt"
 	"reflect"
-	"time"
 )
 
 // AllPtrFieldsNil tests whether all pointer fields in a struct are nil.  This is useful when,
@@ -185,7 +184,7 @@ func StringEqual(a, b *string) bool {
 	return *a == *b
 }
 
-// Float32 returns a pointer to a float32.
+// Float32 returns a pointer to the a float32.
 func Float32(i float32) *float32 {
 	return &i
 }
@@ -215,7 +214,7 @@ func Float32Equal(a, b *float32) bool {
 	return *a == *b
 }
 
-// Float64 returns a pointer to a float64.
+// Float64 returns a pointer to the a float64.
 func Float64(i float64) *float64 {
 	return &i
 }
@@ -244,29 +243,3 @@ func Float64Equal(a, b *float64) bool {
 	}
 	return *a == *b
 }
-
-// Duration returns a pointer to a time.Duration.
-func Duration(d time.Duration) *time.Duration {
-	return &d
-}
-
-// DurationDeref dereferences the time.Duration ptr and returns it if not nil, or else
-// returns def.
-func DurationDeref(ptr *time.Duration, def time.Duration) time.Duration {
-	if ptr != nil {
-		return *ptr
-	}
-	return def
-}
-
-// DurationEqual returns true if both arguments are nil or both arguments
-// dereference to the same value.
-func DurationEqual(a, b *time.Duration) bool {
-	if (a == nil) != (b == nil) {
-		return false
-	}
-	if a == nil {
-		return true
-	}
-	return *a == *b
-}
diff --git a/vendor/k8s.io/utils/strings/slices/slices.go b/vendor/k8s.io/utils/strings/slices/slices.go
deleted file mode 100644
index 8e21838f2..000000000
--- a/vendor/k8s.io/utils/strings/slices/slices.go
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
-Copyright 2021 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package slices defines various functions useful with slices of string type.
-// The goal is to be as close as possible to
-// https://github.com/golang/go/issues/45955. Ideal would be if we can just
-// replace "stringslices" if the "slices" package becomes standard.
-package slices
-
-// Equal reports whether two slices are equal: the same length and all
-// elements equal. If the lengths are different, Equal returns false.
-// Otherwise, the elements are compared in index order, and the
-// comparison stops at the first unequal pair.
-func Equal(s1, s2 []string) bool {
-	if len(s1) != len(s2) {
-		return false
-	}
-	for i, n := range s1 {
-		if n != s2[i] {
-			return false
-		}
-	}
-	return true
-}
-
-// Filter appends to d each element e of s for which keep(e) returns true.
-// It returns the modified d. d may be s[:0], in which case the kept
-// elements will be stored in the same slice.
-// if the slices overlap in some other way, the results are unspecified.
-// To create a new slice with the filtered results, pass nil for d.
-func Filter(d, s []string, keep func(string) bool) []string {
-	for _, n := range s {
-		if keep(n) {
-			d = append(d, n)
-		}
-	}
-	return d
-}
-
-// Contains reports whether v is present in s.
-func Contains(s []string, v string) bool {
-	return Index(s, v) >= 0
-}
-
-// Index returns the index of the first occurrence of v in s, or -1 if
-// not present.
-func Index(s []string, v string) int {
-	// "Contains" may be replaced with "Index(s, v) >= 0":
-	// https://github.com/golang/go/issues/45955#issuecomment-873377947
-	for i, n := range s {
-		if n == v {
-			return i
-		}
-	}
-	return -1
-}
-
-// Functions below are not in https://github.com/golang/go/issues/45955
-
-// Clone returns a new clone of s.
-func Clone(s []string) []string {
-	// https://github.com/go101/go101/wiki/There-is-not-a-perfect-way-to-clone-slices-in-Go
-	if s == nil {
-		return nil
-	}
-	c := make([]string, len(s))
-	copy(c, s)
-	return c
-}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 577ebc2e4..80775a90c 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -260,7 +260,7 @@ github.com/spf13/cobra
 # github.com/spf13/pflag v1.0.5
 ## explicit; go 1.12
 github.com/spf13/pflag
-# github.com/stretchr/testify v1.8.0
+# github.com/stretchr/testify v1.7.5
 ## explicit; go 1.13
 github.com/stretchr/testify/assert
 # go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489
@@ -1030,7 +1030,7 @@ k8s.io/kube-openapi/pkg/handler
 k8s.io/kube-openapi/pkg/schemaconv
 k8s.io/kube-openapi/pkg/util
 k8s.io/kube-openapi/pkg/util/proto
-# k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
+# k8s.io/utils v0.0.0-20211116205334-6203023598ed
 ## explicit; go 1.12
 k8s.io/utils/buffer
 k8s.io/utils/exec
@@ -1039,7 +1039,6 @@ k8s.io/utils/internal/third_party/forked/golang/net
 k8s.io/utils/net
 k8s.io/utils/path
 k8s.io/utils/pointer
-k8s.io/utils/strings/slices
 k8s.io/utils/trace
 # sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30
 ## explicit; go 1.17


Test patch:
diff --git a/pkg/azureclients/armclient/azure_armclient_test.go b/pkg/azureclients/armclient/azure_armclient_test.go
index a94c53657..6ed4dde5d 100644
--- a/pkg/azureclients/armclient/azure_armclient_test.go
+++ b/pkg/azureclients/armclient/azure_armclient_test.go
@@ -73,61 +73,39 @@ func TestSend(t *testing.T) {
 	assert.Equal(t, http.StatusOK, response.StatusCode)
 }
 func TestSendFailureRegionalRetry(t *testing.T) {
-	testcases := []struct {
-		description        string
-		globalServerErrMsg string
-		globalServerCode   int
-	}{
-		{
-			"RegionalRetry",
-			"{\"error\":{\"code\":\"ResourceGroupNotFound\"}}",
-			http.StatusInternalServerError,
-		},
-		{
-			"ReplicationLatency",
-			"{}",
-			http.StatusOK,
-		},
-	}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "GET", r.Method)
+		w.WriteHeader(http.StatusOK)
+		_, err := w.Write([]byte("{}"))
+		assert.NoError(t, err)
+	}))
+	globalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "{\"error\":{\"code\":\"ResourceGroupNotFound\"}}", http.StatusInternalServerError)
+	}))
 
-	for _, tc := range testcases {
-		t.Run(tc.description, func(t *testing.T) {
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				assert.Equal(t, "GET", r.Method)
-				w.WriteHeader(http.StatusOK)
-				_, err := w.Write([]byte("{}"))
-				assert.NoError(t, err)
-			}))
+	azConfig := azureclients.ClientConfig{Backoff: &retry.Backoff{Steps: 3}, UserAgent: "test", Location: "eastus"}
+	armClient := New(nil, azConfig, server.URL, "2019-01-01")
+	targetURL, _ := url.Parse(server.URL)
+	armClient.regionalEndpoint = targetURL.Host
+	pathParameters := map[string]interface{}{
+		"resourceGroupName": autorest.Encode("path", "testgroup"),
+		"subscriptionId":    autorest.Encode("path", "testid"),
+		"resourceName":      autorest.Encode("path", "testname"),
+	}
 
-			globalServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				http.Error(w, tc.globalServerErrMsg, tc.globalServerCode)
-			}))
-			azConfig := azureclients.ClientConfig{Backoff: &retry.Backoff{Steps: 3}, UserAgent: "test", Location: "eastus"}
-			armClient := New(nil, azConfig, server.URL, "2019-01-01")
-			targetURL, _ := url.Parse(server.URL)
-			armClient.regionalEndpoint = targetURL.Host
-			pathParameters := map[string]interface{}{
-				"resourceGroupName": autorest.Encode("path", "testgroup"),
-				"subscriptionId":    autorest.Encode("path", "testid"),
-				"resourceName":      autorest.Encode("path", "testname"),
-			}
-
-			decorators := []autorest.PrepareDecorator{
-				autorest.WithPathParameters(
-					"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/vNets/{resourceName}", pathParameters),
-				autorest.WithBaseURL(globalServer.URL),
-			}
+	decorators := []autorest.PrepareDecorator{
+		autorest.WithPathParameters(
+			"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/vNets/{resourceName}", pathParameters),
+		autorest.WithBaseURL(globalServer.URL),
+	}
 
-			ctx := context.Background()
-			request, err := armClient.PrepareGetRequest(ctx, decorators...)
-			assert.NoError(t, err)
+	ctx := context.Background()
+	request, err := armClient.PrepareGetRequest(ctx, decorators...)
+	assert.NoError(t, err)
 
-			response, rerr := armClient.Send(ctx, request)
-			assert.Nil(t, rerr)
-			assert.Equal(t, http.StatusOK, response.StatusCode)
-			assert.Equal(t, targetURL.Host, response.Request.URL.Host)
-		})
-	}
+	response, rerr := armClient.Send(ctx, request)
+	assert.Nil(t, rerr)
+	assert.Equal(t, http.StatusOK, response.StatusCode)
 }
 
 func TestSendFailure(t *testing.T) {
diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go
index 046434a4d..d08794fb2 100644
--- a/pkg/provider/azure_loadbalancer_test.go
+++ b/pkg/provider/azure_loadbalancer_test.go
@@ -351,78 +351,6 @@ func TestFindRule(t *testing.T) {
 			},
 			expected: true,
 		},
-		{
-			msg: "rule and FrontendIPConfiguration names match should return true",
-			existingRule: []network.LoadBalancingRule{
-				{
-					Name: to.StringPtr("matchName"),
-					LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-						FrontendIPConfiguration: &network.SubResource{ID: to.StringPtr("FrontendIPConfiguration")},
-					},
-				},
-			},
-			curRule: network.LoadBalancingRule{
-				Name: to.StringPtr("matchName"),
-				LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-					FrontendIPConfiguration: &network.SubResource{ID: to.StringPtr("frontendipconfiguration")},
-				},
-			},
-			expected: true,
-		},
-		{
-			msg: "rule names match while FrontendIPConfiguration don't should return false",
-			existingRule: []network.LoadBalancingRule{
-				{
-					Name: to.StringPtr("matchName"),
-					LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-						FrontendIPConfiguration: &network.SubResource{ID: to.StringPtr("FrontendIPConfiguration")},
-					},
-				},
-			},
-			curRule: network.LoadBalancingRule{
-				Name: to.StringPtr("matchName"),
-				LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-					FrontendIPConfiguration: &network.SubResource{ID: to.StringPtr("frontendipconifguration")},
-				},
-			},
-			expected: false,
-		},
-		{
-			msg: "rule and BackendAddressPool names match should return true",
-			existingRule: []network.LoadBalancingRule{
-				{
-					Name: to.StringPtr("matchName"),
-					LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-						BackendAddressPool: &network.SubResource{ID: to.StringPtr("BackendAddressPool")},
-					},
-				},
-			},
-			curRule: network.LoadBalancingRule{
-				Name: to.StringPtr("matchName"),
-				LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-					BackendAddressPool: &network.SubResource{ID: to.StringPtr("backendaddresspool")},
-				},
-			},
-			expected: true,
-		},
-		{
-			msg: "rule and Probe names match should return true",
-			existingRule: []network.LoadBalancingRule{
-				{
-					Name: to.StringPtr("matchName"),
-					LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-						Probe: &network.SubResource{ID: to.StringPtr("Probe")},
-					},
-				},
-			},
-			curRule: network.LoadBalancingRule{
-				Name: to.StringPtr("matchName"),
-				LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-					Probe: &network.SubResource{ID: to.StringPtr("probe")},
-				},
-			},
-			expected: true,
-		},
 	}
 
 	for i, test := range tests {
@@ -1642,15 +1570,14 @@ func TestIsFrontendIPChanged(t *testing.T) {
 			desc: "isFrontendIPChanged shall return false if the service is internal, no loadBalancerIP is given, " +
 				"subnetName == nil and config.PrivateIPAllocationMethod == network.Static",
 			config: network.FrontendIPConfiguration{
-				Name: to.StringPtr("btest1-name"),
+				Name: to.StringPtr("atest1-name"),
 				FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
 					PrivateIPAllocationMethod: network.IPAllocationMethod("static"),
 				},
 			},
-			lbFrontendIPConfigName: "btest1-name",
-			service:                getInternalTestService("test1", 80),
-			expectedFlag:           false,
-			expectedError:          false,
+			service:       getInternalTestService("test1", 80),
+			expectedFlag:  true,
+			expectedError: false,
 		},
 		{
 			desc: "isFrontendIPChanged shall return false if the service is internal, no loadBalancerIP is given, " +
@@ -1683,35 +1610,18 @@ func TestIsFrontendIPChanged(t *testing.T) {
 			expectedError:          false,
 		},
 		{
-			desc: "isFrontendIPChanged shall return false if the service is internal, subnet == nil, " +
-				"loadBalancerIP == config.PrivateIPAddress and config.PrivateIPAllocationMethod != 'static'",
+			desc: "isFrontendIPChanged shall return true if the service is internal, subnet == nil, " +
+				"loadBalancerIP != '' and config.PrivateIPAllocationMethod != 'static'",
 			config: network.FrontendIPConfiguration{
 				Name: to.StringPtr("btest1-name"),
 				FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
 					PrivateIPAllocationMethod: network.IPAllocationMethod("dynamic"),
-					PrivateIPAddress:          to.StringPtr("1.1.1.1"),
-				},
-			},
-			lbFrontendIPConfigName: "btest1-name",
-			service:                getInternalTestService("test1", 80),
-			loadBalancerIP:         "1.1.1.1",
-			expectedFlag:           false,
-			expectedError:          false,
-		},
-		{
-			desc: "isFrontendIPChanged shall return false if the service is internal, subnet == nil, " +
-				"loadBalancerIP == config.PrivateIPAddress and config.PrivateIPAllocationMethod == 'static'",
-			config: network.FrontendIPConfiguration{
-				Name: to.StringPtr("btest1-name"),
-				FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
-					PrivateIPAllocationMethod: network.IPAllocationMethod("static"),
-					PrivateIPAddress:          to.StringPtr("1.1.1.1"),
 				},
 			},
 			lbFrontendIPConfigName: "btest1-name",
 			service:                getInternalTestService("test1", 80),
 			loadBalancerIP:         "1.1.1.1",
-			expectedFlag:           false,
+			expectedFlag:           true,
 			expectedError:          false,
 		},
 		{
@@ -1733,8 +1643,12 @@ func TestIsFrontendIPChanged(t *testing.T) {
 		{
 			desc: "isFrontendIPChanged shall return false if config.PublicIPAddress == nil",
 			config: network.FrontendIPConfiguration{
-				Name:                                    to.StringPtr("btest1-name"),
-				FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{},
+				Name: to.StringPtr("btest1-name"),
+				FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
+					PublicIPAddress: &network.PublicIPAddress{
+						ID: to.StringPtr("pip"),
+					},
+				},
 			},
 			lbFrontendIPConfigName: "btest1-name",
 			service:                getTestService("test1", v1.ProtocolTCP, nil, false, 80),
@@ -2177,15 +2091,6 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
 			probeProtocol:   "Tcp",
 			expectedErr:     true,
 		},
-		{
-			desc:            "getExpectedLBRules should return correct rule when floating ip annotations are added",
-			service:         getTestService("test1", v1.ProtocolTCP, map[string]string{consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true"}, false, 80),
-			loadBalancerSku: "basic",
-			expectedRules: []network.LoadBalancingRule{
-				getFloatingIPTestRule(false, false, 80),
-			},
-			expectedProbes: getDefaultTestProbes("Tcp", ""),
-		},
 	}
 	rules := getDefaultTestRules(true)
 	rules[0].IdleTimeoutInMinutes = to.Int32Ptr(5)
@@ -2366,35 +2271,6 @@ func getHATestRules(enableTCPReset, hasProbe bool, protocol v1.Protocol) []netwo
 	return expectedRules
 }
 
-func getFloatingIPTestRule(enableTCPReset, enableFloatingIP bool, port int32) network.LoadBalancingRule {
-	expectedRules := network.LoadBalancingRule{
-		Name: to.StringPtr(fmt.Sprintf("atest1-TCP-%d", port)),
-		LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-			Protocol: network.TransportProtocol("Tcp"),
-			FrontendIPConfiguration: &network.SubResource{
-				ID: to.StringPtr("frontendIPConfigID"),
-			},
-			BackendAddressPool: &network.SubResource{
-				ID: to.StringPtr("backendPoolID"),
-			},
-			LoadDistribution:     "Default",
-			FrontendPort:         to.Int32Ptr(port),
-			BackendPort:          to.Int32Ptr(getBackendPort(port)),
-			EnableFloatingIP:     to.BoolPtr(enableFloatingIP),
-			DisableOutboundSnat:  to.BoolPtr(false),
-			IdleTimeoutInMinutes: to.Int32Ptr(4),
-			Probe: &network.SubResource{
-				ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
-					fmt.Sprintf("Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-%d", port)),
-			},
-		},
-	}
-	if enableTCPReset {
-		expectedRules.EnableTCPReset = to.BoolPtr(true)
-	}
-	return expectedRules
-}
-
 func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service v1.Service, lbSku string) network.LoadBalancer {
 	caser := cases.Title(language.English)
 	lb := network.LoadBalancer{
@@ -3220,36 +3096,6 @@ func TestReconcileSecurityGroup(t *testing.T) {
 				},
 			},
 		},
-		{
-			desc:    "reconcileSecurityGroup shall create sgs with floating IP disabled",
-			service: getTestService("test1", v1.ProtocolTCP, map[string]string{consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true"}, false, 80),
-			existingSgs: map[string]network.SecurityGroup{"nsg": {
-				Name:                          to.StringPtr("nsg"),
-				SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{},
-			}},
-			lbIP:   to.StringPtr("1.2.3.4"),
-			wantLb: true,
-			expectedSg: &network.SecurityGroup{
-				Name: to.StringPtr("nsg"),
-				SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{
-					SecurityRules: &[]network.SecurityRule{
-						{
-							Name: to.StringPtr("atest1-TCP-80-Internet"),
-							SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
-								Protocol:                   network.SecurityRuleProtocol("Tcp"),
-								SourcePortRange:            to.StringPtr("*"),
-								DestinationPortRange:       to.StringPtr(strconv.Itoa(int(getBackendPort(80)))),
-								SourceAddressPrefix:        to.StringPtr("Internet"),
-								DestinationAddressPrefixes: to.StringSlicePtr([]string{}),
-								Access:                     network.SecurityRuleAccess("Allow"),
-								Priority:                   to.Int32Ptr(500),
-								Direction:                  network.SecurityRuleDirection("Inbound"),
-							},
-						},
-					},
-				},
-			},
-		},
 	}
 
 	for i, test := range testCases {
@@ -3266,7 +3112,7 @@ func TestReconcileSecurityGroup(t *testing.T) {
 				t.Fatalf("TestCase[%d] meets unexpected error: %v", i, err)
 			}
 		}
-		sg, err := az.reconcileSecurityGroup("testCluster", &test.service, test.lbIP, &[]string{}, test.wantLb)
+		sg, err := az.reconcileSecurityGroup("testCluster", &test.service, test.lbIP, test.wantLb)
 		assert.Equal(t, test.expectedSg, sg, "TestCase[%d]: %s", i, test.desc)
 		assert.Equal(t, test.expectedError, err != nil, "TestCase[%d]: %s", i, test.desc)
 	}
@@ -3322,7 +3168,7 @@ func TestReconcileSecurityGroupLoadBalancerSourceRanges(t *testing.T) {
 	mockSGClient := az.SecurityGroupsClient.(*mocksecuritygroupclient.MockInterface)
 	mockSGClient.EXPECT().Get(gomock.Any(), az.ResourceGroup, gomock.Any(), gomock.Any()).Return(existingSg, nil)
 	mockSGClient.EXPECT().CreateOrUpdate(gomock.Any(), az.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
-	sg, err := az.reconcileSecurityGroup("testCluster", &service, lbIP, &[]string{}, true)
+	sg, err := az.reconcileSecurityGroup("testCluster", &service, lbIP, true)
 	assert.NoError(t, err)
 	assert.Equal(t, expectedSg, *sg)
 }
diff --git a/pkg/provider/azure_test.go b/pkg/provider/azure_test.go
index b0c9ebb1d..6ba40202c 100644
--- a/pkg/provider/azure_test.go
+++ b/pkg/provider/azure_test.go
@@ -694,11 +694,11 @@ func TestReconcileSecurityGroupFromAnyDestinationAddressPrefixToLoadBalancerIP(t
 	setMockSecurityGroup(az, ctrl, sg)
 
 	// Simulate a pre-Kubernetes 1.8 NSG, where we do not specify the destination address prefix
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(""), &[]string{}, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(""), true)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -717,7 +717,7 @@ func TestReconcileSecurityGroupDynamicLoadBalancerIP(t *testing.T) {
 	setMockSecurityGroup(az, ctrl, sg)
 
 	dynamicallyAssignedIP := "192.168.0.0"
-	sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(dynamicallyAssignedIP), &[]string{}, true)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(dynamicallyAssignedIP), true)
 	if err != nil {
 		t.Errorf("unexpected error: %q", err)
 	}
@@ -1174,7 +1174,7 @@ func TestReconcileSecurityGroupNewServiceAddsPort(t *testing.T) {
 	lb, _ := az.reconcileLoadBalancer(testClusterName, &svc1, clusterResources.nodes, true)
 	lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb, nil)
 
-	sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbStatus.Ingress[0].IP, &[]string{}, true /* wantLb */)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbStatus.Ingress[0].IP, true /* wantLb */)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -1198,7 +1198,7 @@ func TestReconcileSecurityGroupNewInternalServiceAddsPort(t *testing.T) {
 
 	lb, _ := az.reconcileLoadBalancer(testClusterName, &svc1, clusterResources.nodes, true)
 	lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb, nil)
-	sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbStatus.Ingress[0].IP, &[]string{}, true /* wantLb */)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbStatus.Ingress[0].IP, true /* wantLb */)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -1227,7 +1227,7 @@ func TestReconcileSecurityGroupRemoveService(t *testing.T) {
 	sg := getTestSecurityGroup(az, service1, service2)
 	validateSecurityGroup(t, sg, service1, service2)
 
-	sg, err := az.reconcileSecurityGroup(testClusterName, &service1, &lbStatus.Ingress[0].IP, &[]string{}, false /* wantLb */)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &service1, &lbStatus.Ingress[0].IP, false /* wantLb */)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -1251,7 +1251,7 @@ func TestReconcileSecurityGroupRemoveServiceRemovesPort(t *testing.T) {
 	lb, _ := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true)
 	lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc, lb, nil)
 
-	sg, err := az.reconcileSecurityGroup(testClusterName, &svcUpdated, &lbStatus.Ingress[0].IP, &[]string{}, true /* wantLb */)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &svcUpdated, &lbStatus.Ingress[0].IP, true /* wantLb */)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -1278,7 +1278,7 @@ func TestReconcileSecurityWithSourceRanges(t *testing.T) {
 	lb, _ := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true)
 	lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc, lb, nil)
 
-	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, &lbStatus.Ingress[0].IP, &[]string{}, true /* wantLb */)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, &lbStatus.Ingress[0].IP, true /* wantLb */)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -1314,7 +1314,7 @@ func TestReconcileSecurityGroupEtagMismatch(t *testing.T) {
 	lb, _ := az.reconcileLoadBalancer(testClusterName, &svc1, clusterResources.nodes, true)
 	lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb, nil)
 
-	newSG, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbStatus.Ingress[0].IP, &[]string{}, true /* wantLb */)
+	newSG, err := az.reconcileSecurityGroup(testClusterName, &svc1, &lbStatus.Ingress[0].IP, true /* wantLb */)
 	assert.Nil(t, newSG)
 	assert.Error(t, err)
 
@@ -2328,7 +2328,7 @@ func TestIfServiceSpecifiesSharedRuleAndRuleDoesNotExistItIsCreated(t *testing.T
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -2387,7 +2387,7 @@ func TestIfServiceSpecifiesSharedRuleAndRuleExistsThenTheServicesPortAndAddressA
 	}
 	setMockSecurityGroup(az, ctrl, sg)
 
-	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -2431,12 +2431,12 @@ func TestIfServicesSpecifySharedRuleButDifferentPortsThenSeparateRulesAreCreated
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
@@ -2502,12 +2502,12 @@ func TestIfServicesSpecifySharedRuleButDifferentProtocolsThenSeparateRulesAreCre
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
@@ -2574,12 +2574,12 @@ func TestIfServicesSpecifySharedRuleButDifferentSourceAddressesThenSeparateRules
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
@@ -2648,17 +2648,17 @@ func TestIfServicesSpecifySharedRuleButSomeAreOnDifferentPortsThenRulesAreSepara
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc3: %q", err)
 	}
@@ -2746,19 +2746,19 @@ func TestIfServiceSpecifiesSharedRuleAndServiceIsDeletedThenTheServicesPortAndAd
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
 
 	validateSecurityGroup(t, sg, svc1, svc2)
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, false)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc1: %q", err)
 	}
@@ -2807,24 +2807,24 @@ func TestIfSomeServicesShareARuleAndOneIsDeletedItIsRemovedFromTheRightRule(t *t
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc3: %q", err)
 	}
 
 	validateSecurityGroup(t, sg, svc1, svc2, svc3)
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, false)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc1: %q", err)
 	}
@@ -2915,29 +2915,29 @@ func TestIfServiceSpecifiesSharedRuleAndLastServiceIsDeletedThenRuleIsDeleted(t
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), &[]string{}, true)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), &[]string{}, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc3: %q", err)
 	}
 
 	validateSecurityGroup(t, sg, svc1, svc2, svc3)
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, false)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), &[]string{}, false)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc3: %q", err)
 	}
@@ -3011,7 +3011,7 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) {
 	setMockSecurityGroup(az, ctrl, sg)
 
 	for i, svc := range testServices {
-		_, err := az.reconcileSecurityGroup(testClusterName, &testServices[i], to.StringPtr(svc.Spec.LoadBalancerIP), &[]string{}, true)
+		_, err := az.reconcileSecurityGroup(testClusterName, &testServices[i], to.StringPtr(svc.Spec.LoadBalancerIP), true)
 		if err != nil {
 			t.Errorf("Unexpected error adding svc%d: %q", i+1, err)
 		}
@@ -3103,12 +3103,12 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) {
 		}
 	}
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), &[]string{}, false)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, to.StringPtr(svc5.Spec.LoadBalancerIP), &[]string{}, false)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, to.StringPtr(svc5.Spec.LoadBalancerIP), false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc5: %q", err)
 	}
diff --git a/pkg/provider/azure_utils_test.go b/pkg/provider/azure_utils_test.go
index 48dc6e923..2f5a919f1 100644
--- a/pkg/provider/azure_utils_test.go
+++ b/pkg/provider/azure_utils_test.go
@@ -21,13 +21,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-08-01/network"
 	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/stretchr/testify/assert"
-
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 )
 
@@ -258,59 +255,3 @@ func TestGetServiceAdditionalPublicIPs(t *testing.T) {
 		})
 	}
 }
-
-func TestRemoveDuplicatedSecurityRules(t *testing.T) {
-	for _, testCase := range []struct {
-		description string
-		rules       []network.SecurityRule
-		expected    []network.SecurityRule
-	}{
-		{
-			description: "no duplicated rules",
-			rules: []network.SecurityRule{
-				{
-					Name: to.StringPtr("rule1"),
-				},
-				{
-					Name: to.StringPtr("rule2"),
-				},
-			},
-			expected: []network.SecurityRule{
-				{
-					Name: to.StringPtr("rule1"),
-				},
-				{
-					Name: to.StringPtr("rule2"),
-				},
-			},
-		},
-		{
-			description: "duplicated rules",
-			rules: []network.SecurityRule{
-				{
-					Name: to.StringPtr("rule1"),
-				},
-				{
-					Name: to.StringPtr("rule2"),
-				},
-				{
-					Name: to.StringPtr("rule1"),
-				},
-			},
-			expected: []network.SecurityRule{
-				{
-					Name: to.StringPtr("rule2"),
-				},
-				{
-					Name: to.StringPtr("rule1"),
-				},
-			},
-		},
-	} {
-		t.Run(testCase.description, func(t *testing.T) {
-			rules := testCase.rules
-			rules = removeDuplicatedSecurityRules(rules)
-			assert.Equal(t, testCase.expected, rules)
-		})
-	}
-}
diff --git a/pkg/provider/azure_vmss_cache_test.go b/pkg/provider/azure_vmss_cache_test.go
index 4478bfa2b..2b97adbf7 100644
--- a/pkg/provider/azure_vmss_cache_test.go
+++ b/pkg/provider/azure_vmss_cache_test.go
@@ -17,9 +17,6 @@ limitations under the License.
 package provider
 
 import (
-	"fmt"
-	"net/http"
-	"strings"
 	"sync"
 	"testing"
 
@@ -33,7 +30,6 @@ import (
 	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssclient/mockvmssclient"
 	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssvmclient/mockvmssvmclient"
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
-	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
 func TestExtractVmssVMName(t *testing.T) {
@@ -167,49 +163,3 @@ func TestVMSSVMCacheWithDeletingNodes(t *testing.T) {
 		assert.Equal(t, cloudprovider.InstanceNotFound, err)
 	}
 }
-
-func TestVMSSVMCacheClearedWhenRGDeleted(t *testing.T) {
-	ctrl := gomock.NewController(t)
-	defer ctrl.Finish()
-
-	vmList := []string{"vmssee6c2000000", "vmssee6c2000001", "vmssee6c2000002"}
-	ss, err := NewTestScaleSet(ctrl)
-	assert.NoError(t, err)
-
-	mockVMSSClient := mockvmssclient.NewMockInterface(ctrl)
-	mockVMSSVMClient := mockvmssvmclient.NewMockInterface(ctrl)
-	ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient
-	ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient
-
-	expectedScaleSet := buildTestVMSS(testVMSSName, "vmssee6c2")
-	mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).Times(1)
-
-	expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, vmList, "", false)
-	mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes()
-
-	// validate getting VMSS VM via cache.
-	vm := expectedVMs[0]
-	vmName := to.String(vm.OsProfile.ComputerName)
-	ssName, instanceID, realVM, err := ss.getVmssVM(vmName, azcache.CacheReadTypeDefault)
-	assert.NoError(t, err)
-	assert.Equal(t, "vmss", ssName)
-	assert.Equal(t, to.String(vm.InstanceID), instanceID)
-	assert.Equal(t, &vm, realVM)
-
-	// verify cache has test vmss.
-	cacheKey := strings.ToLower(fmt.Sprintf("%s/%s", "rg", testVMSSName))
-	_, ok := ss.vmssVMCache.Load(cacheKey)
-	assert.Equal(t, true, ok)
-
-	// refresh the cache with error.
-	mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{}, &retry.Error{HTTPStatusCode: http.StatusNotFound}).Times(2)
-	ssName, instanceID, realVM, err = ss.getVmssVM(vmName, azcache.CacheReadTypeForceRefresh)
-	assert.Nil(t, realVM)
-	assert.Equal(t, "", ssName)
-	assert.Equal(t, "", instanceID)
-	assert.Equal(t, cloudprovider.InstanceNotFound, err)
-
-	// verify cache is cleared.
-	_, ok = ss.vmssVMCache.Load(cacheKey)
-	assert.Equal(t, false, ok)
-}
diff --git a/tests/e2e/network/ensureloadbalancer.go b/tests/e2e/network/ensureloadbalancer.go
index 1c954581a..86f586dbd 100644
--- a/tests/e2e/network/ensureloadbalancer.go
+++ b/tests/e2e/network/ensureloadbalancer.go
@@ -51,9 +51,6 @@ var (
 	serviceAnnotationLoadBalancerInternalTrue = map[string]string{
 		consts.ServiceAnnotationLoadBalancerInternal: "true",
 	}
-	serviceAnnotationDisableLoadBalancerFloatingIP = map[string]string{
-		consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true",
-	}
 )
 
 var _ = Describe("Ensure LoadBalancer", func() {
@@ -520,41 +517,6 @@ var _ = Describe("Ensure LoadBalancer", func() {
 		err = waitForNodesInLBBackendPool(tc, publicIP, len(nodes))
 		Expect(err).NotTo(HaveOccurred())
 	})
-
-	It("should support disabling floating IP in load balancer rule with kubernetes service annotations", func() {
-		By("creating a public IP with tags")
-		ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4]
-		pip := defaultPublicIPAddress(ipName)
-		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip)
-		Expect(err).NotTo(HaveOccurred())
-		targetIP := to.String(pip.IPAddress)
-		utils.Logf("created pip with address %s", targetIP)
-
-		By("creating a service referencing the public IP")
-		service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationDisableLoadBalancerFloatingIP, labels, ns.Name, ports)
-		service = updateServiceBalanceIP(service, false, targetIP)
-		_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
-		Expect(err).NotTo(HaveOccurred())
-		ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "")
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ip).To(Equal(targetIP))
-
-		defer func() {
-			By("cleaning up")
-			err = utils.DeleteService(cs, ns.Name, testServiceName)
-			Expect(err).NotTo(HaveOccurred())
-			err = utils.DeletePIPWithRetry(tc, ipName, "")
-			Expect(err).NotTo(HaveOccurred())
-		}()
-
-		By("testing if floating IP disabled in load balancer rule")
-		lb := getAzureLoadBalancerFromPIP(tc, ip, tc.GetResourceGroup(), "")
-		lbRules := lb.LoadBalancingRules
-		Expect(len(*lbRules)).To(Equal(len(ports)))
-		for _, lbRule := range *lbRules {
-			Expect(to.Bool(lbRule.EnableFloatingIP)).To(BeFalse())
-		}
-	})
 })
 
 func waitForNodesInLBBackendPool(tc *utils.AzureTestClient, ip string, expectedNum int) error {
diff --git a/tests/e2e/network/network_security_group.go b/tests/e2e/network/network_security_group.go
index 4d7314c76..10842a970 100644
--- a/tests/e2e/network/network_security_group.go
+++ b/tests/e2e/network/network_security_group.go
@@ -30,7 +30,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/utils/strings/slices"
 
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 	"sigs.k8s.io/cloud-provider-azure/tests/e2e/utils"
@@ -220,50 +219,6 @@ var _ = Describe("Network security group", func() {
 		Expect(found).To(BeTrue())
 	})
 
-	It("should support service annotation `service.beta.kubernetes.io/azure-disable-load-balancer-floating-ip`", func() {
-		By("Creating a public IP with tags")
-		ipName := basename + "-public-IP-disable-floating-ip"
-		pip := defaultPublicIPAddress(ipName)
-		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip)
-		Expect(err).NotTo(HaveOccurred())
-		targetIP := to.String(pip.IPAddress)
-		utils.Logf("created pip with address %s", targetIP)
-
-		By("Creating a test load balancer service with floating IP disabled")
-		annotation := map[string]string{
-			consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true",
-		}
-		service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports)
-		service = updateServiceBalanceIP(service, false, targetIP)
-		_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
-		Expect(err).NotTo(HaveOccurred())
-		ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "")
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ip).To(Equal(targetIP))
-
-		defer func() {
-			By("cleaning up")
-			err = utils.DeleteService(cs, ns.Name, serviceName)
-			Expect(err).NotTo(HaveOccurred())
-			err = utils.DeletePIPWithRetry(tc, ipName, "")
-			Expect(err).NotTo(HaveOccurred())
-		}()
-
-		By("Checking if the LoadBalancer's public IP is included in the network security rule's DestinationAddressPrefixes")
-		nsgs, err := tc.GetClusterSecurityGroups()
-		Expect(err).NotTo(HaveOccurred())
-		for _, nsg := range nsgs {
-			if nsg.SecurityRules == nil {
-				continue
-			}
-
-			for _, securityRules := range *nsg.SecurityRules {
-				contains := slices.Contains(*securityRules.DestinationAddressPrefixes, targetIP)
-				Expect(contains).To(BeFalse())
-			}
-		}
-	})
-
 })
 
 func validateUnsharedSecurityRuleExists(nsgs []aznetwork.SecurityGroup, ip string, port string) bool {
diff --git a/vendor/github.com/stretchr/testify/assert/assertion_format.go b/vendor/github.com/stretchr/testify/assert/assertion_format.go
index 7880b8f94..27e2420ed 100644
--- a/vendor/github.com/stretchr/testify/assert/assertion_format.go
+++ b/vendor/github.com/stretchr/testify/assert/assertion_format.go
@@ -736,16 +736,6 @@ func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta tim
 	return WithinDuration(t, expected, actual, delta, append([]interface{}{msg}, args...)...)
 }
 
-// WithinRangef asserts that a time is within a time range (inclusive).
-//
-//   assert.WithinRangef(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted")
-func WithinRangef(t TestingT, actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
-	return WithinRange(t, actual, start, end, append([]interface{}{msg}, args...)...)
-}
-
 // YAMLEqf asserts that two YAML strings are equivalent.
 func YAMLEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) bool {
 	if h, ok := t.(tHelper); ok {
diff --git a/vendor/github.com/stretchr/testify/assert/assertion_forward.go b/vendor/github.com/stretchr/testify/assert/assertion_forward.go
index 339515b8b..d9ea368d0 100644
--- a/vendor/github.com/stretchr/testify/assert/assertion_forward.go
+++ b/vendor/github.com/stretchr/testify/assert/assertion_forward.go
@@ -1461,26 +1461,6 @@ func (a *Assertions) WithinDurationf(expected time.Time, actual time.Time, delta
 	return WithinDurationf(a.t, expected, actual, delta, msg, args...)
 }
 
-// WithinRange asserts that a time is within a time range (inclusive).
-//
-//   a.WithinRange(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second))
-func (a *Assertions) WithinRange(actual time.Time, start time.Time, end time.Time, msgAndArgs ...interface{}) bool {
-	if h, ok := a.t.(tHelper); ok {
-		h.Helper()
-	}
-	return WithinRange(a.t, actual, start, end, msgAndArgs...)
-}
-
-// WithinRangef asserts that a time is within a time range (inclusive).
-//
-//   a.WithinRangef(time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second), "error message %s", "formatted")
-func (a *Assertions) WithinRangef(actual time.Time, start time.Time, end time.Time, msg string, args ...interface{}) bool {
-	if h, ok := a.t.(tHelper); ok {
-		h.Helper()
-	}
-	return WithinRangef(a.t, actual, start, end, msg, args...)
-}
-
 // YAMLEq asserts that two YAML strings are equivalent.
 func (a *Assertions) YAMLEq(expected string, actual string, msgAndArgs ...interface{}) bool {
 	if h, ok := a.t.(tHelper); ok {
diff --git a/vendor/github.com/stretchr/testify/assert/assertions.go b/vendor/github.com/stretchr/testify/assert/assertions.go
index fa1245b18..580fdea4c 100644
--- a/vendor/github.com/stretchr/testify/assert/assertions.go
+++ b/vendor/github.com/stretchr/testify/assert/assertions.go
@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"path/filepath"
 	"reflect"
 	"regexp"
 	"runtime"
@@ -145,8 +144,7 @@ func CallerInfo() []string {
 		if len(parts) > 1 {
 			dir := parts[len(parts)-2]
 			if (dir != "assert" && dir != "mock" && dir != "require") || file == "mock_test.go" {
-				path, _ := filepath.Abs(file)
-				callers = append(callers, fmt.Sprintf("%s:%d", path, line))
+				callers = append(callers, fmt.Sprintf("%s:%d", file, line))
 			}
 		}
 
@@ -818,6 +816,7 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok
 		return true // we consider nil to be equal to the nil set
 	}
 
+	subsetValue := reflect.ValueOf(subset)
 	defer func() {
 		if e := recover(); e != nil {
 			ok = false
@@ -827,32 +826,14 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok
 	listKind := reflect.TypeOf(list).Kind()
 	subsetKind := reflect.TypeOf(subset).Kind()
 
-	if listKind != reflect.Array && listKind != reflect.Slice && listKind != reflect.Map {
+	if listKind != reflect.Array && listKind != reflect.Slice {
 		return Fail(t, fmt.Sprintf("%q has an unsupported type %s", list, listKind), msgAndArgs...)
 	}
 
-	if subsetKind != reflect.Array && subsetKind != reflect.Slice && listKind != reflect.Map {
+	if subsetKind != reflect.Array && subsetKind != reflect.Slice {
 		return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...)
 	}
 
-	subsetValue := reflect.ValueOf(subset)
-	if subsetKind == reflect.Map && listKind == reflect.Map {
-		listValue := reflect.ValueOf(list)
-		subsetKeys := subsetValue.MapKeys()
-
-		for i := 0; i < len(subsetKeys); i++ {
-			subsetKey := subsetKeys[i]
-			subsetElement := subsetValue.MapIndex(subsetKey).Interface()
-			listElement := listValue.MapIndex(subsetKey).Interface()
-
-			if !ObjectsAreEqual(subsetElement, listElement) {
-				return Fail(t, fmt.Sprintf("\"%s\" does not contain \"%s\"", list, subsetElement), msgAndArgs...)
-			}
-		}
-
-		return true
-	}
-
 	for i := 0; i < subsetValue.Len(); i++ {
 		element := subsetValue.Index(i).Interface()
 		ok, found := containsElement(list, element)
@@ -879,6 +860,7 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{})
 		return Fail(t, "nil is the empty set which is a subset of every set", msgAndArgs...)
 	}
 
+	subsetValue := reflect.ValueOf(subset)
 	defer func() {
 		if e := recover(); e != nil {
 			ok = false
@@ -888,32 +870,14 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{})
 	listKind := reflect.TypeOf(list).Kind()
 	subsetKind := reflect.TypeOf(subset).Kind()
 
-	if listKind != reflect.Array && listKind != reflect.Slice && listKind != reflect.Map {
+	if listKind != reflect.Array && listKind != reflect.Slice {
 		return Fail(t, fmt.Sprintf("%q has an unsupported type %s", list, listKind), msgAndArgs...)
 	}
 
-	if subsetKind != reflect.Array && subsetKind != reflect.Slice && listKind != reflect.Map {
+	if subsetKind != reflect.Array && subsetKind != reflect.Slice {
 		return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...)
 	}
 
-	subsetValue := reflect.ValueOf(subset)
-	if subsetKind == reflect.Map && listKind == reflect.Map {
-		listValue := reflect.ValueOf(list)
-		subsetKeys := subsetValue.MapKeys()
-
-		for i := 0; i < len(subsetKeys); i++ {
-			subsetKey := subsetKeys[i]
-			subsetElement := subsetValue.MapIndex(subsetKey).Interface()
-			listElement := listValue.MapIndex(subsetKey).Interface()
-
-			if !ObjectsAreEqual(subsetElement, listElement) {
-				return true
-			}
-		}
-
-		return Fail(t, fmt.Sprintf("%q is a subset of %q", subset, list), msgAndArgs...)
-	}
-
 	for i := 0; i < subsetValue.Len(); i++ {
 		element := subsetValue.Index(i).Interface()
 		ok, found := containsElement(list, element)
@@ -1146,27 +1110,6 @@ func WithinDuration(t TestingT, expected, actual time.Time, delta time.Duration,
 	return true
 }
 
-// WithinRange asserts that a time is within a time range (inclusive).
-//
-//   assert.WithinRange(t, time.Now(), time.Now().Add(-time.Second), time.Now().Add(time.Second))
-func WithinRange(t TestingT, actual, start, end time.Time, msgAndArgs ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
-
-	if end.Before(start) {
-		return Fail(t, "Start should be before end", msgAndArgs...)
-	}
-
-	if actual.Before(start) {
-		return Fail(t, fmt.Sprintf("Time %v expected to be in time range %v to %v, but is before the range", actual, start, end), msgAndArgs...)
-	} else if actual.After(end) {
-		return Fail(t, fmt.Sprintf("Time %v expected to be in time range %v to %v, but is after the range", actual, start, end), msgAndArgs...)
-	}
-
-	return true
-}
-
 func toFloat(x interface{}) (float64, bool) {
 	var xf float64
 	xok := true