Repository: kubernetes-sigs/cloud-provider-azure
Pull request #1125 (2022-02-10 02:15:01)

[release-1.23] feat: support changing LB backend pool type between nodeIP and nodeIP…
…Configuration in existing clusters

(cherry picked from commit 12ba8146fe0febebbdb78234a371e5a284d149b9)

<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?

<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

/kind feature

#### What this PR does / why we need it:

Cherry pick of #1034 to release-1.23: feat: support changing LB backend pool type between nodeIP and nodeIPConfiguration in existing clusters


#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
feat: support changing LB backend pool type between nodeIP and nodeIP…
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```



Patch:
diff --git a/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go b/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go
index ca17e982f..8f7d4bd99 100644
--- a/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go
+++ b/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go
@@ -492,6 +492,49 @@ func (c *Client) createOrUpdateLBBackendPool(ctx context.Context, resourceGroupN
 	return nil
 }
 
+// DeleteLBBackendPool deletes a LoadBalancer backend pool by name.
+func (c *Client) DeleteLBBackendPool(ctx context.Context, resourceGroupName, loadBalancerName, backendPoolName string) *retry.Error {
+	mc := metrics.NewMetricContext("load_balancers", "delete_backend_pool", resourceGroupName, c.subscriptionID, "")
+
+	// Report errors if the client is rate limited.
+	if !c.rateLimiterWriter.TryAccept() {
+		mc.RateLimitedCount()
+		return retry.GetRateLimitError(true, "LBDeleteBackendPool")
+	}
+
+	// Report errors if the client is throttled.
+	if c.RetryAfterWriter.After(time.Now()) {
+		mc.ThrottledCount()
+		rerr := retry.GetThrottlingError("LBDeleteBackendPool", "client throttled", c.RetryAfterWriter)
+		return rerr
+	}
+
+	rerr := c.deleteLBBackendPool(ctx, resourceGroupName, loadBalancerName, backendPoolName)
+	mc.Observe(rerr)
+	if rerr != nil {
+		if rerr.IsThrottled() {
+			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
+			c.RetryAfterWriter = rerr.RetryAfter
+		}
+
+		return rerr
+	}
+
+	return nil
+}
+
+func (c *Client) deleteLBBackendPool(ctx context.Context, resourceGroupName, loadBalancerName, backendPoolName string) *retry.Error {
+	resourceID := armclient.GetChildResourceID(
+		c.subscriptionID,
+		resourceGroupName,
+		"Microsoft.Network/loadBalancers",
+		loadBalancerName,
+		"backendAddressPools",
+		backendPoolName,
+	)
+	return c.armClient.DeleteResource(ctx, resourceID, "")
+}
+
 func (c *Client) createOrUpdateBackendPoolResponder(resp *http.Response) (*network.BackendAddressPool, *retry.Error) {
 	result := &network.BackendAddressPool{}
 	err := autorest.Respond(
diff --git a/pkg/azureclients/loadbalancerclient/interface.go b/pkg/azureclients/loadbalancerclient/interface.go
index 519715158..8ea9e9804 100644
--- a/pkg/azureclients/loadbalancerclient/interface.go
+++ b/pkg/azureclients/loadbalancerclient/interface.go
@@ -50,4 +50,7 @@ type Interface interface {
 
 	// Delete deletes a LoadBalancer by name.
 	Delete(ctx context.Context, resourceGroupName string, loadBalancerName string) *retry.Error
+
+	// DeleteLBBackendPool deletes a LoadBalancer backend pool by name.
+	DeleteLBBackendPool(ctx context.Context, resourceGroupName, loadBalancerName, backendPoolName string) *retry.Error
 }
diff --git a/pkg/azureclients/loadbalancerclient/mockloadbalancerclient/interface.go b/pkg/azureclients/loadbalancerclient/mockloadbalancerclient/interface.go
index 7e45c8aa9..989ea1152 100644
--- a/pkg/azureclients/loadbalancerclient/mockloadbalancerclient/interface.go
+++ b/pkg/azureclients/loadbalancerclient/mockloadbalancerclient/interface.go
@@ -16,7 +16,7 @@
 //
 
 // Code generated by MockGen. DO NOT EDIT.
-// Source: /go/src/sigs.k8s.io/cloud-provider-azure/pkg/azureclients/loadbalancerclient/interface.go
+// Source: pkg/azureclients/loadbalancerclient/interface.go
 
 // Package mockloadbalancerclient is a generated GoMock package.
 package mockloadbalancerclient
@@ -53,36 +53,6 @@ func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
-// Get mocks base method.
-func (m *MockInterface) Get(ctx context.Context, resourceGroupName, loadBalancerName, expand string) (network.LoadBalancer, *retry.Error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, loadBalancerName, expand)
-	ret0, _ := ret[0].(network.LoadBalancer)
-	ret1, _ := ret[1].(*retry.Error)
-	return ret0, ret1
-}
-
-// Get indicates an expected call of Get.
-func (mr *MockInterfaceMockRecorder) Get(ctx, resourceGroupName, loadBalancerName, expand interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), ctx, resourceGroupName, loadBalancerName, expand)
-}
-
-// List mocks base method.
-func (m *MockInterface) List(ctx context.Context, resourceGroupName string) ([]network.LoadBalancer, *retry.Error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "List", ctx, resourceGroupName)
-	ret0, _ := ret[0].([]network.LoadBalancer)
-	ret1, _ := ret[1].(*retry.Error)
-	return ret0, ret1
-}
-
-// List indicates an expected call of List.
-func (mr *MockInterfaceMockRecorder) List(ctx, resourceGroupName interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockInterface)(nil).List), ctx, resourceGroupName)
-}
-
 // CreateOrUpdate mocks base method.
 func (m *MockInterface) CreateOrUpdate(ctx context.Context, resourceGroupName, loadBalancerName string, parameters network.LoadBalancer, etag string) *retry.Error {
 	m.ctrl.T.Helper()
@@ -124,3 +94,47 @@ func (mr *MockInterfaceMockRecorder) Delete(ctx, resourceGroupName, loadBalancer
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockInterface)(nil).Delete), ctx, resourceGroupName, loadBalancerName)
 }
+
+// DeleteLBBackendPool mocks base method.
+func (m *MockInterface) DeleteLBBackendPool(ctx context.Context, resourceGroupName, loadBalancerName, backendPoolName string) *retry.Error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteLBBackendPool", ctx, resourceGroupName, loadBalancerName, backendPoolName)
+	ret0, _ := ret[0].(*retry.Error)
+	return ret0
+}
+
+// DeleteLBBackendPool indicates an expected call of DeleteLBBackendPool.
+func (mr *MockInterfaceMockRecorder) DeleteLBBackendPool(ctx, resourceGroupName, loadBalancerName, backendPoolName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLBBackendPool", reflect.TypeOf((*MockInterface)(nil).DeleteLBBackendPool), ctx, resourceGroupName, loadBalancerName, backendPoolName)
+}
+
+// Get mocks base method.
+func (m *MockInterface) Get(ctx context.Context, resourceGroupName, loadBalancerName, expand string) (network.LoadBalancer, *retry.Error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, loadBalancerName, expand)
+	ret0, _ := ret[0].(network.LoadBalancer)
+	ret1, _ := ret[1].(*retry.Error)
+	return ret0, ret1
+}
+
+// Get indicates an expected call of Get.
+func (mr *MockInterfaceMockRecorder) Get(ctx, resourceGroupName, loadBalancerName, expand interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), ctx, resourceGroupName, loadBalancerName, expand)
+}
+
+// List mocks base method.
+func (m *MockInterface) List(ctx context.Context, resourceGroupName string) ([]network.LoadBalancer, *retry.Error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "List", ctx, resourceGroupName)
+	ret0, _ := ret[0].([]network.LoadBalancer)
+	ret1, _ := ret[1].(*retry.Error)
+	return ret0, ret1
+}
+
+// List indicates an expected call of List.
+func (mr *MockInterfaceMockRecorder) List(ctx, resourceGroupName interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockInterface)(nil).List), ctx, resourceGroupName)
+}
diff --git a/pkg/provider/azure_backoff.go b/pkg/provider/azure_backoff.go
index c236fca7b..97313b793 100644
--- a/pkg/provider/azure_backoff.go
+++ b/pkg/provider/azure_backoff.go
@@ -293,6 +293,34 @@ func (az *Cloud) CreateOrUpdateLBBackendPool(lbName string, backendPool network.
 	return rerr.Error()
 }
 
+func (az *Cloud) DeleteLBBackendPool(lbName, backendPoolName string) error {
+	ctx, cancel := getContextWithCancel()
+	defer cancel()
+
+	klog.V(4).Infof("DeleteLBBackendPool: deleting backend pool %s in LB %s", backendPoolName, lbName)
+	rerr := az.LoadBalancerClient.DeleteLBBackendPool(ctx, az.getLoadBalancerResourceGroup(), lbName, backendPoolName)
+	if rerr == nil {
+		// Invalidate the cache right after updating
+		_ = az.lbCache.Delete(lbName)
+		return nil
+	}
+
+	// Invalidate the cache because ETAG precondition mismatch.
+	if rerr.HTTPStatusCode == http.StatusPreconditionFailed {
+		klog.V(3).Infof("LoadBalancer cache for %s is cleanup because of http.StatusPreconditionFailed", lbName)
+		_ = az.lbCache.Delete(lbName)
+	}
+
+	retryErrorMessage := rerr.Error().Error()
+	// Invalidate the cache because another new operation has canceled the current request.
+	if strings.Contains(strings.ToLower(retryErrorMessage), consts.OperationCanceledErrorMessage) {
+		klog.V(3).Infof("LoadBalancer cache for %s is cleanup because CreateOrUpdate is canceled by another operation", lbName)
+		_ = az.lbCache.Delete(lbName)
+	}
+
+	return rerr.Error()
+}
+
 // ListManagedLBs invokes az.LoadBalancerClient.List and filter out
 // those that are not managed by cloud provider azure or not associated to a managed VMSet.
 func (az *Cloud) ListManagedLBs(service *v1.Service, nodes []*v1.Node, clusterName string) ([]network.LoadBalancer, error) {
diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go
index b4fd9aa88..7afcca4c8 100644
--- a/pkg/provider/azure_loadbalancer.go
+++ b/pkg/provider/azure_loadbalancer.go
@@ -379,7 +379,7 @@ func (az *Cloud) cleanOrphanedLoadBalancer(lb *network.LoadBalancer, existingLBs
 
 // safeDeleteLoadBalancer deletes the load balancer after decoupling it from the vmSet
 func (az *Cloud) safeDeleteLoadBalancer(lb network.LoadBalancer, clusterName, vmSetName string, service *v1.Service) *retry.Error {
-	if strings.EqualFold(az.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypeNodeIPConfiguration) {
+	if isLBBackendPoolTypeIPConfig(service, &lb, clusterName) {
 		lbBackendPoolID := az.getBackendPoolID(to.String(lb.Name), az.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, service))
 		err := az.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools, true)
 		if err != nil {
diff --git a/pkg/provider/azure_loadbalancer_backendpool.go b/pkg/provider/azure_loadbalancer_backendpool.go
index 2d3c6f86c..379bd7d62 100644
--- a/pkg/provider/azure_loadbalancer_backendpool.go
+++ b/pkg/provider/azure_loadbalancer_backendpool.go
@@ -137,7 +137,6 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 	}
 
 	foundBackendPool := false
-	wantLb := true
 	changed := false
 	lbName := *lb.Name
 
@@ -146,11 +145,30 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 	lbBackendPoolID := bc.getBackendPoolID(lbName, bc.getLoadBalancerResourceGroup(), lbBackendPoolName)
 	vmSetName := bc.mapLoadBalancerNameToVMSet(lbName, clusterName)
 
-	for _, bp := range newBackendPools {
+	for i := len(newBackendPools) - 1; i >= 0; i-- {
+		bp := newBackendPools[i]
 		if strings.EqualFold(*bp.Name, lbBackendPoolName) {
-			klog.V(10).Infof("bc.ReconcileBackendPools for service (%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb)
+			klog.V(10).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found wanted backendpool. not adding anything", serviceName)
 			foundBackendPool = true
 
+			// If the LB backend pool type is configured from nodeIP or podIP
+			// to nodeIPConfiguration, we need to decouple the VM NICs from the LB
+			// before attaching nodeIPs/podIPs to the LB backend pool.
+			if bp.BackendAddressPoolPropertiesFormat != nil &&
+				(bp.BackendIPConfigurations == nil || len(*bp.BackendIPConfigurations) == 0) {
+				// It is not allowed to change the type of the backend pool, so we delete it
+				// and create a new empty backend pool.
+				if err := bc.DeleteLBBackendPool(lbName, to.String(bp.Name)); err != nil {
+					klog.Errorf("bc.ReconcileBackendPools for service (%s): failed to DeleteLBBackendPool: %s", serviceName, err.Error())
+					return false, false, err
+				}
+				newBackendPools = append(newBackendPools[:i], newBackendPools[i+1:]...)
+				lb.BackendAddressPools = &newBackendPools
+				lb.Etag = nil
+				foundBackendPool = false
+				break
+			}
+
 			var backendIPConfigurationsToBeDeleted []network.InterfaceIPConfiguration
 			if bp.BackendAddressPoolPropertiesFormat != nil && bp.BackendIPConfigurations != nil {
 				for _, ipConf := range *bp.BackendIPConfigurations {
@@ -170,7 +188,7 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 						return false, false, err
 					}
 					if shouldExcludeLoadBalancer {
-						klog.V(2).Infof("bc.ReconcileBackendPools for service (%s)(%t): lb backendpool - found unwanted node %s, decouple it from the LB %s", serviceName, wantLb, nodeName, lbName)
+						klog.V(2).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found unwanted node %s, decouple it from the LB %s", serviceName, nodeName, lbName)
 						// construct a backendPool that only contains the IP config of the node to be deleted
 						backendIPConfigurationsToBeDeleted = append(backendIPConfigurationsToBeDeleted, network.InterfaceIPConfiguration{ID: to.StringPtr(ipConfID)})
 					}
@@ -193,7 +211,7 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 			}
 			break
 		} else {
-			klog.V(10).Infof("bc.ReconcileBackendPools for service (%s)(%t): lb backendpool - found unmanaged backendpool %s", serviceName, wantLb, *bp.Name)
+			klog.V(10).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found unmanaged backendpool %s", serviceName, *bp.Name)
 		}
 	}
 
@@ -357,27 +375,53 @@ func (bi *backendPoolTypeNodeIP) CleanupVMSetFromBackendPoolByCondition(slb *net
 
 func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, service *v1.Service, lb *network.LoadBalancer) (bool, bool, error) {
 	var newBackendPools []network.BackendAddressPool
-	var err error
 	if lb.BackendAddressPools != nil {
 		newBackendPools = *lb.BackendAddressPools
 	}
 
 	foundBackendPool := false
-	wantLb := true
 	changed := false
 	lbName := *lb.Name
 	serviceName := getServiceName(service)
 	lbBackendPoolName := getBackendPoolName(clusterName, service)
+	vmSetName := bi.mapLoadBalancerNameToVMSet(lbName, clusterName)
+	lbBackendPoolID := bi.getBackendPoolID(to.String(lb.Name), bi.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, service))
 
-	for i, bp := range newBackendPools {
+	for i := len(newBackendPools) - 1; i >= 0; i-- {
+		bp := newBackendPools[i]
 		if strings.EqualFold(*bp.Name, lbBackendPoolName) {
-			klog.V(10).Infof("bi.ReconcileBackendPools for service (%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb)
+			klog.V(10).Infof("bi.ReconcileBackendPools for service (%s): found wanted backendpool. not adding anything", serviceName)
 			foundBackendPool = true
 
+			// If the LB backend pool type is configured from nodeIPConfiguration
+			// to nodeIP, we need to decouple the VM NICs from the LB
+			// before attaching nodeIPs/podIPs to the LB backend pool.
+			if bp.BackendAddressPoolPropertiesFormat != nil &&
+				bp.BackendIPConfigurations != nil &&
+				len(*bp.BackendIPConfigurations) > 0 {
+				// It is not allowed to change the type of the backend pool, so we delete it
+				// and create a new empty backend pool. Note that for IP configuration based
+				// backend pools, we must decouple it from the vmSets before deleting.
+				klog.V(2).Infof("bi.ReconcileBackendPools for service (%s): ensuring the LB is decoupled from the VMSet", serviceName)
+				if err := bi.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolID, vmSetName, lb.BackendAddressPools, true); err != nil {
+					klog.Errorf("bi.ReconcileBackendPools for service (%s): failed to EnsureBackendPoolDeleted: %s", serviceName, err.Error())
+					return false, false, err
+				}
+				if err := bi.DeleteLBBackendPool(lbName, to.String(bp.Name)); err != nil {
+					klog.Errorf("bi.ReconcileBackendPools for service (%s): failed to DeleteLBBackendPool: %s", serviceName, err.Error())
+					return false, false, err
+				}
+				newBackendPools = append(newBackendPools[:i], newBackendPools[i+1:]...)
+				lb.BackendAddressPools = &newBackendPools
+				lb.Etag = nil
+				foundBackendPool = false
+				break
+			}
+
 			var nodeIPAddressesToBeDeleted []string
 			for nodeName := range bi.excludeLoadBalancerNodes {
 				for ip := range bi.nodePrivateIPs[nodeName] {
-					klog.V(2).Infof("bi.ReconcileBackendPools for service (%s)(%t): lb backendpool - found unwanted node private IP %s, decouple it from the LB %s", serviceName, wantLb, ip, lbName)
+					klog.V(2).Infof("bi.ReconcileBackendPools for service (%s): found unwanted node private IP %s, decoupling it from the LB %s", serviceName, ip, lbName)
 					nodeIPAddressesToBeDeleted = append(nodeIPAddressesToBeDeleted, ip)
 				}
 			}
@@ -386,13 +430,13 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi
 				if updated {
 					(*lb.BackendAddressPools)[i] = bp
 					if err := bi.CreateOrUpdateLBBackendPool(lbName, bp); err != nil {
-						return false, false, fmt.Errorf("bi.ReconcileBackendPools for service (%s)(%t): lb backendpool - failed to update backend pool %s for load balancer %s: %w", serviceName, wantLb, lbBackendPoolName, lbName, err)
+						return false, false, fmt.Errorf("bi.ReconcileBackendPools for service (%s): lb backendpool - failed to update backend pool %s for load balancer %s: %w", serviceName, lbBackendPoolName, lbName, err)
 					}
 				}
 			}
 			break
 		} else {
-			klog.V(10).Infof("bi.ReconcileBackendPools for service (%s)(%t): lb backendpool - found unmanaged backendpool %s", serviceName, wantLb, *bp.Name)
+			klog.V(10).Infof("bi.ReconcileBackendPools for service (%s): found unmanaged backendpool %s", serviceName, *bp.Name)
 		}
 	}
 
@@ -402,7 +446,7 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi
 		changed = true
 	}
 
-	return isBackendPoolPreConfigured, changed, err
+	return isBackendPoolPreConfigured, changed, nil
 }
 
 func newBackendPool(lb *network.LoadBalancer, isBackendPoolPreConfigured bool, preConfiguredBackendPoolLoadBalancerTypes, serviceName, lbBackendPoolName string) bool {
diff --git a/pkg/provider/azure_utils.go b/pkg/provider/azure_utils.go
index 1bde0b561..b1177b188 100644
--- a/pkg/provider/azure_utils.go
+++ b/pkg/provider/azure_utils.go
@@ -260,3 +260,19 @@ func getNodePrivateIPAddresses(node *v1.Node) []string {
 
 	return addresses
 }
+
+func isLBBackendPoolTypeIPConfig(service *v1.Service, lb *network.LoadBalancer, clusterName string) bool {
+	if lb == nil || lb.LoadBalancerPropertiesFormat == nil || lb.BackendAddressPools == nil {
+		klog.V(4).Infof("isLBBackendPoolTypeIPConfig: no backend pools in the LB %s", to.String(lb.Name))
+		return false
+	}
+	lbBackendPoolName := getBackendPoolName(clusterName, service)
+	for _, bp := range *lb.BackendAddressPools {
+		if strings.EqualFold(to.String(bp.Name), lbBackendPoolName) {
+			return bp.BackendAddressPoolPropertiesFormat != nil &&
+				bp.BackendIPConfigurations != nil &&
+				len(*bp.BackendIPConfigurations) != 0
+		}
+	}
+	return false
+}


Test patch:
diff --git a/pkg/provider/azure_backoff_test.go b/pkg/provider/azure_backoff_test.go
index cf5fb5c33..f57c4329c 100644
--- a/pkg/provider/azure_backoff_test.go
+++ b/pkg/provider/azure_backoff_test.go
@@ -621,3 +621,34 @@ func TestCreateOrUpdateLBBackendPool(t *testing.T) {
 		assert.Equal(t, tc.expectedErr, err != nil)
 	}
 }
+
+func TestDeleteLBBackendPool(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	for _, tc := range []struct {
+		description string
+		deleteErr   *retry.Error
+		expectedErr bool
+	}{
+		{
+			description: "DeleteLBBackendPool should not report an error if the api call succeeds",
+		},
+		{
+			description: "DeleteLBBackendPool should report an error if the api call fails",
+			deleteErr: &retry.Error{
+				HTTPStatusCode: http.StatusPreconditionFailed,
+				RawError:       errors.New(consts.OperationCanceledErrorMessage),
+			},
+			expectedErr: true,
+		},
+	} {
+		az := GetTestCloud(ctrl)
+		lbClient := mockloadbalancerclient.NewMockInterface(ctrl)
+		lbClient.EXPECT().DeleteLBBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.deleteErr)
+		az.LoadBalancerClient = lbClient
+
+		err := az.DeleteLBBackendPool("kubernetes", "kubernetes")
+		assert.Equal(t, tc.expectedErr, err != nil)
+	}
+}
diff --git a/pkg/provider/azure_loadbalancer_backendpool_test.go b/pkg/provider/azure_loadbalancer_backendpool_test.go
index da6192d05..5993906ce 100644
--- a/pkg/provider/azure_loadbalancer_backendpool_test.go
+++ b/pkg/provider/azure_loadbalancer_backendpool_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package provider
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -315,6 +316,28 @@ func TestReconcileBackendPoolsNodeIPConfig(t *testing.T) {
 	assert.True(t, changed)
 }
 
+func TestReconcileBackendPoolsNodeIPToIPConfig(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	lb := buildLBWithVMIPs(testClusterName, []string{"10.0.0.1", "10.0.0.2"})
+	mockLBClient := mockloadbalancerclient.NewMockInterface(ctrl)
+	mockLBClient.EXPECT().DeleteLBBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(retry.NewError(false, fmt.Errorf("delete LB backend pool error")))
+
+	az := GetTestCloud(ctrl)
+	az.LoadBalancerClient = mockLBClient
+	bc := newBackendPoolTypeNodeIPConfig(az)
+	svc := getTestService("test", v1.ProtocolTCP, nil, false, 80)
+	_, _, err := bc.ReconcileBackendPools(testClusterName, &svc, lb)
+	assert.Contains(t, err.Error(), "delete LB backend pool error")
+
+	lb = buildLBWithVMIPs(testClusterName, []string{"10.0.0.1", "10.0.0.2"})
+	mockLBClient.EXPECT().DeleteLBBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
+	_, _, err = bc.ReconcileBackendPools(testClusterName, &svc, lb)
+	assert.NoError(t, err)
+	assert.Empty(t, (*lb.BackendAddressPools)[0].LoadBalancerBackendAddresses)
+}
+
 func TestReconcileBackendPoolsNodeIP(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -379,6 +402,48 @@ func TestReconcileBackendPoolsNodeIP(t *testing.T) {
 	assert.True(t, changed)
 }
 
+func TestReconcileBackendPoolsNodeIPConfigToIP(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	lb := buildDefaultTestLB(testClusterName, []string{
+		"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1",
+		"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool2-00000000-nic-1/ipConfigurations/ipconfig1",
+	})
+	mockVMSet := NewMockVMSet(ctrl)
+	mockVMSet.EXPECT().EnsureBackendPoolDeleted(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("delete LB backend pool error"))
+	mockVMSet.EXPECT().GetPrimaryVMSetName().Return("k8s-agentpool1-00000000").AnyTimes()
+
+	az := GetTestCloud(ctrl)
+	az.VMSet = mockVMSet
+	//az.LoadBalancerClient = mockLBClient
+	bi := newBackendPoolTypeNodeIP(az)
+	svc := getTestService("test", v1.ProtocolTCP, nil, false, 80)
+	_, _, err := bi.ReconcileBackendPools(testClusterName, &svc, &lb)
+	assert.Contains(t, err.Error(), "delete LB backend pool error")
+
+	lb = buildDefaultTestLB(testClusterName, []string{
+		"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1",
+		"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool2-00000000-nic-1/ipConfigurations/ipconfig1",
+	})
+	mockLBClient := mockloadbalancerclient.NewMockInterface(ctrl)
+	mockLBClient.EXPECT().DeleteLBBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(retry.NewError(false, fmt.Errorf("error2")))
+	mockVMSet.EXPECT().EnsureBackendPoolDeleted(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
+	az.LoadBalancerClient = mockLBClient
+	_, _, err = bi.ReconcileBackendPools(testClusterName, &svc, &lb)
+	assert.Contains(t, err.Error(), "error2")
+
+	lb = buildDefaultTestLB(testClusterName, []string{
+		"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1",
+		"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool2-00000000-nic-1/ipConfigurations/ipconfig1",
+	})
+	mockLBClient.EXPECT().DeleteLBBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
+	mockVMSet.EXPECT().EnsureBackendPoolDeleted(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
+	_, _, err = bi.ReconcileBackendPools(testClusterName, &svc, &lb)
+	assert.NoError(t, err)
+	assert.Empty(t, (*lb.BackendAddressPools)[0].LoadBalancerBackendAddresses)
+}
+
 func TestRemoveNodeIPAddressFromBackendPool(t *testing.T) {
 	nodeIPAddresses := []string{"1.2.3.4", "4.3.2.1"}
 	backendPool := network.BackendAddressPool{
diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go
index d94798212..0372911bb 100644
--- a/pkg/provider/azure_loadbalancer_test.go
+++ b/pkg/provider/azure_loadbalancer_test.go
@@ -4064,7 +4064,7 @@ func TestCheckLoadBalancerResourcesConflicted(t *testing.T) {
 
 func buildLBWithVMIPs(clusterName string, vmIPs []string) *network.LoadBalancer {
 	lb := network.LoadBalancer{
-		Name: to.StringPtr("clusterName"),
+		Name: to.StringPtr(clusterName),
 		LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{
 			BackendAddressPools: &[]network.BackendAddressPool{
 				{