Repository: kubernetes-sigs/cloud-provider-azure
Pull request #4042 (2023-06-06 06:59:32)

[release-1.27] feat: support migrate from NIC-based to IP-based backend pool by migration API by setting `"enableMigrateToIPBasedBackendPoolAPI": true`
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?

<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

/kind feature

#### What this PR does / why we need it:
Support migrate from NIC-based to IP-based backend pool by migration API by setting `"enableMigrateToIpBasedBackendPoolAPI": true`. By using this API, there will be no downtime during the migration.

#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
feat: Support migrate from NIC-based to IP-based backend pool by migration API by setting `"enableMigrateToIpBasedBackendPoolAPI": true`. By using this API, there will be no downtime during the migration.
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```


k8s-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubernetes-sigs/cloud-provider-azure/pull/4042#" title="Author self-approved">nilo19</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fcloud-provider-azure).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubernetes-sigs/cloud-provider-azure/blob/release-1.27/OWNERS)~~ [nilo19]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go b/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go
index 5c41f79b8..f641d2b44 100644
--- a/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go
+++ b/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go
@@ -55,6 +55,10 @@ type Client struct {
 	RetryAfterWriter time.Time
 }
 
+type backendPoolsToBeMigrated struct {
+	BackendPoolNames []string `json:"pools"`
+}
+
 // New creates a new LoadBalancer client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
 	baseURI := config.ResourceManagerEndpoint
@@ -421,6 +425,69 @@ func (page LoadBalancerListResultPage) Values() []network.LoadBalancer {
 	return *page.lblr.Value
 }
 
+// GetLBBackendPool gets a LoadBalancer backend pool.
+func (c *Client) GetLBBackendPool(ctx context.Context, resourceGroupName string, loadBalancerName string, backendPoolName string, expand string) (network.BackendAddressPool, *retry.Error) {
+	mc := metrics.NewMetricContext("load_balancers", "get_backend_pool", resourceGroupName, c.subscriptionID, "")
+
+	// Report errors if the client is rate limited.
+	if !c.rateLimiterReader.TryAccept() {
+		mc.RateLimitedCount()
+		return network.BackendAddressPool{}, retry.GetRateLimitError(false, "LBGet")
+	}
+
+	// Report errors if the client is throttled.
+	if c.RetryAfterReader.After(time.Now()) {
+		mc.ThrottledCount()
+		rerr := retry.GetThrottlingError("LBBackendPoolGet", "client throttled", c.RetryAfterReader)
+		return network.BackendAddressPool{}, rerr
+	}
+
+	result, rerr := c.getLBBackendPool(ctx, resourceGroupName, loadBalancerName, backendPoolName, expand)
+	mc.Observe(rerr)
+	if rerr != nil {
+		if rerr.IsThrottled() {
+			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
+			c.RetryAfterReader = rerr.RetryAfter
+		}
+
+		return result, rerr
+	}
+
+	return result, nil
+}
+
+// getLBBackendPool gets a LoadBalancer backend pool.
+func (c *Client) getLBBackendPool(ctx context.Context, resourceGroupName string, loadBalancerName string, backendPoolName string, expand string) (network.BackendAddressPool, *retry.Error) {
+	resourceID := armclient.GetChildResourceID(
+		c.subscriptionID,
+		resourceGroupName,
+		lbResourceType,
+		loadBalancerName,
+		"backendAddressPools",
+		backendPoolName,
+	)
+	result := network.BackendAddressPool{}
+
+	response, rerr := c.armClient.GetResourceWithExpandQuery(ctx, resourceID, expand)
+	defer c.armClient.CloseResponse(ctx, response)
+	if rerr != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "loadbalancer.backendpool.get.request", resourceID, rerr.Error())
+		return result, rerr
+	}
+
+	err := autorest.Respond(
+		response,
+		azure.WithErrorUnlessStatusCode(http.StatusOK),
+		autorest.ByUnmarshallingJSON(&result))
+	if err != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "loadbalancer.backendpool.get.respond", resourceID, err)
+		return result, retry.GetError(response, err)
+	}
+
+	result.Response = autorest.Response{Response: response}
+	return result, nil
+}
+
 // CreateOrUpdateBackendPools creates or updates a LoadBalancer backend pool.
 func (c *Client) CreateOrUpdateBackendPools(ctx context.Context, resourceGroupName string, loadBalancerName string, backendPoolName string, parameters network.BackendAddressPool, etag string) *retry.Error {
 	mc := metrics.NewMetricContext("load_balancers", "create_or_update_backend_pools", resourceGroupName, c.subscriptionID, "")
@@ -470,14 +537,14 @@ func (c *Client) createOrUpdateLBBackendPool(ctx context.Context, resourceGroupN
 	response, rerr := c.armClient.PutResource(ctx, resourceID, parameters, decorators...)
 	defer c.armClient.CloseResponse(ctx, response)
 	if rerr != nil {
-		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "loadbalancerbackendpool.put.request", resourceID, rerr.Error())
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "loadbalancer.backendpool.put.request", resourceID, rerr.Error())
 		return rerr
 	}
 
 	if response != nil && response.StatusCode != http.StatusNoContent {
 		_, rerr = c.createOrUpdateBackendPoolResponder(response)
 		if rerr != nil {
-			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "loadbalancerbackendpool.put.respond", resourceID, rerr.Error())
+			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "loadbalancer.backendpool.put.respond", resourceID, rerr.Error())
 			return rerr
 		}
 	}
@@ -537,3 +604,56 @@ func (c *Client) createOrUpdateBackendPoolResponder(resp *http.Response) (*netwo
 	result.Response = autorest.Response{Response: resp}
 	return result, retry.GetError(resp, err)
 }
+
+// MigrateToIPBasedBackendPool migrates a NIC-based backend pool to IP-based.
+func (c *Client) MigrateToIPBasedBackendPool(ctx context.Context, resourceGroupName string, loadBalancerName string, backendPoolNames []string) *retry.Error {
+	mc := metrics.NewMetricContext("load_balancers", "migrate_to_ip_based_backend_pool", resourceGroupName, c.subscriptionID, "")
+
+	// Report errors if the client is rate limited.
+	if !c.rateLimiterWriter.TryAccept() {
+		mc.RateLimitedCount()
+		return retry.GetRateLimitError(true, "LBMigrateToIPBasedBackendPool")
+	}
+
+	// Report errors if the client is throttled.
+	if c.RetryAfterWriter.After(time.Now()) {
+		mc.ThrottledCount()
+		rerr := retry.GetThrottlingError("LBMigrateToIPBasedBackendPool", "client throttled", c.RetryAfterWriter)
+		return rerr
+	}
+
+	parameters := backendPoolsToBeMigrated{
+		BackendPoolNames: backendPoolNames,
+	}
+	rerr := c.migrateToIPBasedBackendPool(ctx, resourceGroupName, loadBalancerName, parameters)
+	mc.Observe(rerr)
+	if rerr != nil {
+		if rerr.IsThrottled() {
+			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
+			c.RetryAfterWriter = rerr.RetryAfter
+		}
+
+		return rerr
+	}
+
+	return nil
+}
+
+func (c *Client) migrateToIPBasedBackendPool(ctx context.Context, resourceGroupName string, loadBalancerName string, parameters backendPoolsToBeMigrated) *retry.Error {
+	resourceID := armclient.GetResourceID(
+		c.subscriptionID,
+		resourceGroupName,
+		lbResourceType,
+		loadBalancerName,
+	)
+
+	response, rerr := c.armClient.PostResource(ctx, resourceID, "migrateToIpBased", parameters, map[string]interface{}{})
+	defer c.armClient.CloseResponse(ctx, response)
+	if rerr != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "loadbalancerbackendpool.migrate.request", resourceID, rerr.Error())
+		return rerr
+	}
+
+	klog.Infof("Response: %v", response)
+	return nil
+}
diff --git a/pkg/azureclients/loadbalancerclient/interface.go b/pkg/azureclients/loadbalancerclient/interface.go
index 7a1a88f7d..6f0feb22f 100644
--- a/pkg/azureclients/loadbalancerclient/interface.go
+++ b/pkg/azureclients/loadbalancerclient/interface.go
@@ -51,6 +51,12 @@ type Interface interface {
 	// Delete deletes a LoadBalancer by name.
 	Delete(ctx context.Context, resourceGroupName string, loadBalancerName string) *retry.Error
 
+	// GetLBBackendPool gets a LoadBalancer backend pool.
+	GetLBBackendPool(ctx context.Context, resourceGroupName string, loadBalancerName string, backendPoolName string, expand string) (network.BackendAddressPool, *retry.Error)
+
 	// DeleteLBBackendPool deletes a LoadBalancer backend pool by name.
 	DeleteLBBackendPool(ctx context.Context, resourceGroupName, loadBalancerName, backendPoolName string) *retry.Error
+
+	// MigrateToIPBasedBackendPool migrates a NIC-based backend pool to IP-based.
+	MigrateToIPBasedBackendPool(ctx context.Context, resourceGroupName string, loadBalancerName string, backendPoolNames []string) *retry.Error
 }
diff --git a/pkg/azureclients/loadbalancerclient/mockloadbalancerclient/interface.go b/pkg/azureclients/loadbalancerclient/mockloadbalancerclient/interface.go
index 9397a9782..c32c8a230 100644
--- a/pkg/azureclients/loadbalancerclient/mockloadbalancerclient/interface.go
+++ b/pkg/azureclients/loadbalancerclient/mockloadbalancerclient/interface.go
@@ -124,6 +124,21 @@ func (mr *MockInterfaceMockRecorder) Get(ctx, resourceGroupName, loadBalancerNam
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), ctx, resourceGroupName, loadBalancerName, expand)
 }
 
+// GetLBBackendPool mocks base method.
+func (m *MockInterface) GetLBBackendPool(ctx context.Context, resourceGroupName, loadBalancerName, backendPoolName, expand string) (network.BackendAddressPool, *retry.Error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetLBBackendPool", ctx, resourceGroupName, loadBalancerName, backendPoolName, expand)
+	ret0, _ := ret[0].(network.BackendAddressPool)
+	ret1, _ := ret[1].(*retry.Error)
+	return ret0, ret1
+}
+
+// GetLBBackendPool indicates an expected call of GetLBBackendPool.
+func (mr *MockInterfaceMockRecorder) GetLBBackendPool(ctx, resourceGroupName, loadBalancerName, backendPoolName, expand interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLBBackendPool", reflect.TypeOf((*MockInterface)(nil).GetLBBackendPool), ctx, resourceGroupName, loadBalancerName, backendPoolName, expand)
+}
+
 // List mocks base method.
 func (m *MockInterface) List(ctx context.Context, resourceGroupName string) ([]network.LoadBalancer, *retry.Error) {
 	m.ctrl.T.Helper()
@@ -138,3 +153,17 @@ func (mr *MockInterfaceMockRecorder) List(ctx, resourceGroupName interface{}) *g
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockInterface)(nil).List), ctx, resourceGroupName)
 }
+
+// MigrateToIPBasedBackendPool mocks base method.
+func (m *MockInterface) MigrateToIPBasedBackendPool(ctx context.Context, resourceGroupName, loadBalancerName string, backendPoolNames []string) *retry.Error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "MigrateToIPBasedBackendPool", ctx, resourceGroupName, loadBalancerName, backendPoolNames)
+	ret0, _ := ret[0].(*retry.Error)
+	return ret0
+}
+
+// MigrateToIPBasedBackendPool indicates an expected call of MigrateToIPBasedBackendPool.
+func (mr *MockInterfaceMockRecorder) MigrateToIPBasedBackendPool(ctx, resourceGroupName, loadBalancerName, backendPoolNames interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MigrateToIPBasedBackendPool", reflect.TypeOf((*MockInterface)(nil).MigrateToIPBasedBackendPool), ctx, resourceGroupName, loadBalancerName, backendPoolNames)
+}
diff --git a/pkg/provider/azure.go b/pkg/provider/azure.go
index cb1155903..e470c41a2 100644
--- a/pkg/provider/azure.go
+++ b/pkg/provider/azure.go
@@ -259,6 +259,12 @@ type Config struct {
 	PutVMSSVMBatchSize int `json:"putVMSSVMBatchSize" yaml:"putVMSSVMBatchSize"`
 	// PrivateLinkServiceResourceGroup determines the specific resource group of the private link services user want to use
 	PrivateLinkServiceResourceGroup string `json:"privateLinkServiceResourceGroup,omitempty" yaml:"privateLinkServiceResourceGroup,omitempty"`
+
+	// EnableMigrateToIPBasedBackendPoolAPI uses the migration API to migrate from NIC-based to IP-based backend pool.
+	// The migration API can provide a migration from NIC-based to IP-based backend pool without service downtime.
+	// If the API is not used, the migration will be done by decoupling all nodes on the backend pool and then re-attaching
+	// node IPs, which will introduce service downtime. The downtime increases with the number of nodes in the backend pool.
+	EnableMigrateToIPBasedBackendPoolAPI bool `json:"enableMigrateToIPBasedBackendPoolAPI" yaml:"enableMigrateToIPBasedBackendPoolAPI"`
 }
 
 type InitSecretConfig struct {
diff --git a/pkg/provider/azure_backoff.go b/pkg/provider/azure_backoff.go
index f855be538..68b2b58cb 100644
--- a/pkg/provider/azure_backoff.go
+++ b/pkg/provider/azure_backoff.go
@@ -17,12 +17,14 @@ limitations under the License.
 package provider
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
@@ -652,3 +654,54 @@ func (az *Cloud) CreateOrUpdateSubnet(service *v1.Service, subnet network.Subnet
 
 	return nil
 }
+
+// MigrateToIPBasedBackendPoolAndWaitForCompletion use the migration API to migrate from
+// NIC-based to IP-based LB backend pools. It also makes sure the number of IP addresses
+// in the backend pools is expected.
+func (az *Cloud) MigrateToIPBasedBackendPoolAndWaitForCompletion(
+	lbName string, backendPoolNames []string, nicsCountMap map[string]int,
+) error {
+	if rerr := az.LoadBalancerClient.MigrateToIPBasedBackendPool(context.Background(), az.ResourceGroup, lbName, backendPoolNames); rerr != nil {
+		backendPoolNamesStr := strings.Join(backendPoolNames, ",")
+		klog.Errorf("MigrateToIPBasedBackendPoolAndWaitForCompletion: Failed to migrate to IP based backend pool for lb %s, backend pool %s: %s", lbName, backendPoolNamesStr, rerr.Error().Error())
+		return rerr.Error()
+	}
+
+	succeeded := make(map[string]bool)
+	for bpName := range nicsCountMap {
+		succeeded[bpName] = false
+	}
+
+	err := wait.PollImmediate(5*time.Second, 10*time.Minute, func() (done bool, err error) {
+		for bpName, nicsCount := range nicsCountMap {
+			if succeeded[bpName] {
+				continue
+			}
+
+			bp, rerr := az.LoadBalancerClient.GetLBBackendPool(context.Background(), az.ResourceGroup, lbName, bpName, "")
+			if rerr != nil {
+				klog.Errorf("MigrateToIPBasedBackendPoolAndWaitForCompletion: Failed to get backend pool %s for lb %s: %s", bpName, lbName, rerr.Error().Error())
+				return false, rerr.Error()
+			}
+
+			if countIPsOnBackendPool(bp) != nicsCount {
+				klog.V(4).Infof("MigrateToIPBasedBackendPoolAndWaitForCompletion: Expected IPs %s, current IPs %d, will retry in 5s", nicsCount, countIPsOnBackendPool(bp))
+				return false, nil
+			}
+			succeeded[bpName] = true
+		}
+		return true, nil
+	})
+
+	if err != nil {
+		if errors.Is(err, wait.ErrWaitTimeout) {
+			klog.Warningf("MigrateToIPBasedBackendPoolAndWaitForCompletion: Timeout waiting for migration to IP based backend pool for lb %s, backend pool %s", lbName, strings.Join(backendPoolNames, ","))
+			return nil
+		}
+
+		klog.Errorf("MigrateToIPBasedBackendPoolAndWaitForCompletion: Failed to wait for migration to IP based backend pool for lb %s, backend pool %s: %s", lbName, strings.Join(backendPoolNames, ","), err.Error())
+		return err
+	}
+
+	return nil
+}
diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go
index 2466ec09b..6fc94411e 100644
--- a/pkg/provider/azure_loadbalancer.go
+++ b/pkg/provider/azure_loadbalancer.go
@@ -1560,7 +1560,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 
 	// reconcile the load balancer's backend pool configuration.
 	if wantLb {
-		preConfig, changed, err := az.LoadBalancerBackendPool.ReconcileBackendPools(clusterName, service, lb)
+		preConfig, changed, shouldRefreshLB, err := az.LoadBalancerBackendPool.ReconcileBackendPools(clusterName, service, lb)
 		if err != nil {
 			return lb, err
 		}
@@ -1568,6 +1568,16 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 			dirtyLb = true
 		}
 		isBackendPoolPreConfigured = preConfig
+
+		// If the LB is changed, refresh it to avoid etag mismatch error
+		// later when create or update the LB.
+		if shouldRefreshLB {
+			klog.V(4).Infof("reconcileLoadBalancer for service(%s): refreshing load balancer %s", serviceName, lbName)
+			lb, _, err = az.getAzureLoadBalancer(lbName, azcache.CacheReadTypeForceRefresh)
+			if err != nil {
+				return lb, fmt.Errorf("reconcileLoadBalancer for service (%s): failed to get load balancer %s: %w", serviceName, lbName, err)
+			}
+		}
 	}
 
 	// reconcile the load balancer's frontend IP configurations.
diff --git a/pkg/provider/azure_loadbalancer_backendpool.go b/pkg/provider/azure_loadbalancer_backendpool.go
index e7f75d29e..052b81bc9 100644
--- a/pkg/provider/azure_loadbalancer_backendpool.go
+++ b/pkg/provider/azure_loadbalancer_backendpool.go
@@ -52,7 +52,7 @@ type BackendPool interface {
 
 	// ReconcileBackendPools creates the inbound backend pool if it is not existed, and removes nodes that are supposed to be
 	// excluded from the load balancers.
-	ReconcileBackendPools(clusterName string, service *v1.Service, lb *network.LoadBalancer) (bool, bool, error)
+	ReconcileBackendPools(clusterName string, service *v1.Service, lb *network.LoadBalancer) (bool, bool, bool, error)
 
 	// GetBackendPrivateIPs returns the private IPs of LoadBalancer's backend pool
 	GetBackendPrivateIPs(clusterName string, service *v1.Service, lb *network.LoadBalancer) ([]string, []string)
@@ -164,7 +164,7 @@ func (bc *backendPoolTypeNodeIPConfig) CleanupVMSetFromBackendPoolByCondition(sl
 	return slb, nil
 }
 
-func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string, service *v1.Service, lb *network.LoadBalancer) (bool, bool, error) {
+func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string, service *v1.Service, lb *network.LoadBalancer) (bool, bool, bool, error) {
 	var newBackendPools []network.BackendAddressPool
 	var err error
 	if lb.BackendAddressPools != nil {
@@ -214,7 +214,7 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 					bp.VirtualNetwork = nil
 					if err := bc.CreateOrUpdateLBBackendPool(lbName, bp); err != nil {
 						klog.Errorf("bc.ReconcileBackendPools for service (%s): failed to cleanup IP based backend pool %s: %s", serviceName, lbBackendPoolNames[isIPv6], err.Error())
-						return false, false, fmt.Errorf("bc.ReconcileBackendPools for service (%s): failed to cleanup IP based backend pool %s: %w", serviceName, lbBackendPoolNames[isIPv6], err)
+						return false, false, false, fmt.Errorf("bc.ReconcileBackendPools for service (%s): failed to cleanup IP based backend pool %s: %w", serviceName, lbBackendPoolNames[isIPv6], err)
 					}
 					newBackendPools[i] = bp
 					lb.BackendAddressPools = &newBackendPools
@@ -232,7 +232,7 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 							klog.V(2).Infof("bc.ReconcileBackendPools for service (%s): vm not found for ipConfID %s", serviceName, ipConfID)
 							bipConfigNotFound = append(bipConfigNotFound, ipConf)
 						} else {
-							return false, false, err
+							return false, false, false, err
 						}
 					}
 
@@ -243,7 +243,7 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 					shouldExcludeLoadBalancer, err := bc.ShouldNodeExcludedFromLoadBalancer(nodeName)
 					if err != nil {
 						klog.Errorf("bc.ReconcileBackendPools: ShouldNodeExcludedFromLoadBalancer(%s) failed with error: %v", nodeName, err)
-						return false, false, err
+						return false, false, false, err
 					}
 					if shouldExcludeLoadBalancer {
 						klog.V(2).Infof("bc.ReconcileBackendPools for service (%s): lb backendpool - found unwanted node %s, decouple it from the LB %s", serviceName, nodeName, lbName)
@@ -271,7 +271,7 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 		// decouple the backendPool from the node
 		updated, err := bc.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolIDsSlice, vmSetName, &backendpoolToBeDeleted, false)
 		if err != nil {
-			return false, false, err
+			return false, false, false, err
 		}
 		if updated {
 			shouldRefreshLB = true
@@ -281,7 +281,7 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 	if shouldRefreshLB {
 		lb, _, err = bc.getAzureLoadBalancer(lbName, cache.CacheReadTypeForceRefresh)
 		if err != nil {
-			return false, false, fmt.Errorf("bc.ReconcileBackendPools for service (%s): failed to get loadbalancer %s: %w", serviceName, lbName, err)
+			return false, false, false, fmt.Errorf("bc.ReconcileBackendPools for service (%s): failed to get loadbalancer %s: %w", serviceName, lbName, err)
 		}
 	}
 
@@ -302,7 +302,7 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string,
 	}
 
 	isOperationSucceeded = true
-	return isBackendPoolPreConfigured, changed, err
+	return isBackendPoolPreConfigured, changed, false, err
 }
 
 func getBackendIPConfigurationsToBeDeleted(
@@ -555,7 +555,7 @@ func (bi *backendPoolTypeNodeIP) CleanupVMSetFromBackendPoolByCondition(slb *net
 	return slb, nil
 }
 
-func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, service *v1.Service, lb *network.LoadBalancer) (bool, bool, error) {
+func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, service *v1.Service, lb *network.LoadBalancer) (bool, bool, bool, error) {
 	var newBackendPools []network.BackendAddressPool
 	if lb.BackendAddressPools != nil {
 		newBackendPools = *lb.BackendAddressPools
@@ -577,17 +577,32 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi
 	isBackendPoolPreConfigured := bi.isBackendPoolPreConfigured(service)
 
 	mc := metrics.NewMetricContext("services", "migrate_to_ip_based_backend_pool", bi.ResourceGroup, bi.getNetworkResourceSubscriptionID(), serviceName)
-	var err error
-	bpIdxes := []int{}
-	lbBackendPoolIDsSlice := []string{}
+
+	var (
+		err                   error
+		bpIdxes               []int
+		lbBackendPoolIDsSlice []string
+	)
+	nicsCountMap := make(map[string]int)
 	for i := len(newBackendPools) - 1; i >= 0; i-- {
 		bp := newBackendPools[i]
 		found, isIPv6 := isLBBackendPoolsExisting(lbBackendPoolNames, bp.Name)
 		if found {
 			bpIdxes = append(bpIdxes, i)
-			klog.V(10).Infof("bi.ReconcileBackendPools for service (%s): found wanted backendpool. not adding anything", serviceName)
+			klog.V(10).Infof("bi.ReconcileBackendPools for service (%s): found wanted backendpool. Not adding anything", serviceName)
 			foundBackendPools[isIPv6] = true
 			lbBackendPoolIDsSlice = append(lbBackendPoolIDsSlice, lbBackendPoolIDs[isIPv6])
+
+			if nicsCount := countNICsOnBackendPool(bp); nicsCount > 0 {
+				nicsCountMap[pointer.StringDeref(bp.Name, "")] = nicsCount
+				klog.V(4).Infof(
+					"bi.ReconcileBackendPools for service(%s): found NIC-based backendpool %s with %d NICs, will migrate to IP-based",
+					serviceName,
+					pointer.StringDeref(bp.Name, ""),
+					nicsCount,
+				)
+				isMigration = true
+			}
 		} else {
 			klog.V(10).Infof("bi.ReconcileBackendPools for service (%s): found unmanaged backendpool %s", serviceName, *bp.Name)
 		}
@@ -598,14 +613,45 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi
 		// If the LB backend pool type is configured from nodeIPConfiguration
 		// to nodeIP, we need to decouple the VM NICs from the LB
 		// before attaching nodeIPs/podIPs to the LB backend pool.
+		// If the migration API is enabled, we use the migration API to decouple
+		// the VM NICs from the LB. Then we manually decouple the VMSS
+		// and its VMs from the LB by EnsureBackendPoolDeleted. These manual operations
+		// cannot be omitted because we use the VMSS manual upgrade policy.
+		// If the migration API is not enabled, we manually decouple the VM NICs and
+		// the VMSS from the LB by EnsureBackendPoolDeleted. If no NIC-based backend
+		// pool is found (it is not a migration scenario), EnsureBackendPoolDeleted would be a no-op.
+		if isMigration && bi.EnableMigrateToIPBasedBackendPoolAPI {
+			var backendPoolNames []string
+			for _, id := range lbBackendPoolIDsSlice {
+				name, err := getLBNameFromBackendPoolID(id)
+				if err != nil {
+					klog.Errorf("bi.ReconcileBackendPools for service (%s): failed to get LB name from backend pool ID: %s", serviceName, err.Error())
+					return false, false, false, err
+				}
+				backendPoolNames = append(backendPoolNames, name)
+			}
+
+			if err := bi.MigrateToIPBasedBackendPoolAndWaitForCompletion(lbName, backendPoolNames, nicsCountMap); err != nil {
+				backendPoolNamesStr := strings.Join(backendPoolNames, ",")
+				klog.Errorf("Failed to migrate to IP based backend pool for lb %s, backend pool %s: %s", lbName, backendPoolNamesStr, err.Error())
+				return false, false, false, err
+			}
+		}
+
+		// EnsureBackendPoolDeleted is useful in the following scenarios:
+		// 1. Migrate from NIC-based to IP-based backend pool if the migration
+		// API is not enabled.
+		// 2. Migrate from NIC-based to IP-based backend pool when the migration
+		// API is enabled. This is needed because since we use the manual upgrade
+		// policy on VMSS so the migration API will not change the VMSS and VMSS
+		// VMs during the migration.
+		// 3. Decouple vmss from the lb if the backend pool is empty when using
+		// ip-based LB. Ref: https://github.com/kubernetes-sigs/cloud-provider-azure/pull/2829.
 		klog.V(2).Infof("bi.ReconcileBackendPools for service (%s) and vmSet (%s): ensuring the LB is decoupled from the VMSet", serviceName, vmSetName)
 		shouldRefreshLB, err = bi.VMSet.EnsureBackendPoolDeleted(service, lbBackendPoolIDsSlice, vmSetName, lb.BackendAddressPools, true)
 		if err != nil {
 			klog.Errorf("bi.ReconcileBackendPools for service (%s): failed to EnsureBackendPoolDeleted: %s", serviceName, err.Error())
-			return false, false, err
-		}
-		if shouldRefreshLB {
-			isMigration = true
+			return false, false, false, err
 		}
 
 		for _, i := range bpIdxes {
@@ -650,19 +696,14 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi
 			if updated {
 				(*lb.BackendAddressPools)[i] = bp
 				if err := bi.CreateOrUpdateLBBackendPool(lbName, bp); err != nil {
-					return false, false, fmt.Errorf("bi.ReconcileBackendPools for service (%s): lb backendpool - failed to update backend pool %s for load balancer %s: %w", serviceName, pointer.StringDeref(bp.Name, ""), lbName, err)
+					return false, false, false, fmt.Errorf("bi.ReconcileBackendPools for service (%s): lb backendpool - failed to update backend pool %s for load balancer %s: %w", serviceName, pointer.StringDeref(bp.Name, ""), lbName, err)
 				}
 				shouldRefreshLB = true
 			}
 		}
 	}
 
-	if shouldRefreshLB {
-		lb, _, err = bi.getAzureLoadBalancer(lbName, cache.CacheReadTypeForceRefresh)
-		if err != nil {
-			return false, false, fmt.Errorf("bi.ReconcileBackendPools for service (%s): failed to get load balancer %s: %w", serviceName, lbName, err)
-		}
-	}
+	shouldRefreshLB = shouldRefreshLB || isMigration
 
 	for _, ipFamily := range service.Spec.IPFamilies {
 		if foundBackendPools[ipFamily == v1.IPv6Protocol] {
@@ -681,7 +722,7 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi
 	}
 
 	isOperationSucceeded = true
-	return isBackendPoolPreConfigured, changed, nil
+	return isBackendPoolPreConfigured, changed, shouldRefreshLB, nil
 }
 
 func (bi *backendPoolTypeNodeIP) GetBackendPrivateIPs(clusterName string, service *v1.Service, lb *network.LoadBalancer) ([]string, []string) {
diff --git a/pkg/provider/azure_mock_loadbalancer_backendpool.go b/pkg/provider/azure_mock_loadbalancer_backendpool.go
index 474eac6be..9d1911995 100644
--- a/pkg/provider/azure_mock_loadbalancer_backendpool.go
+++ b/pkg/provider/azure_mock_loadbalancer_backendpool.go
@@ -21,48 +21,33 @@ import (
 
 	network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
 	gomock "github.com/golang/mock/gomock"
-
 	v1 "k8s.io/api/core/v1"
 )
 
-// MockBackendPool is a mock of BackendPool interface
+// MockBackendPool is a mock of BackendPool interface.
 type MockBackendPool struct {
 	ctrl     *gomock.Controller
 	recorder *MockBackendPoolMockRecorder
 }
 
-// MockBackendPoolMockRecorder is the mock recorder for MockBackendPool
+// MockBackendPoolMockRecorder is the mock recorder for MockBackendPool.
 type MockBackendPoolMockRecorder struct {
 	mock *MockBackendPool
 }
 
-// NewMockBackendPool creates a new mock instance
+// NewMockBackendPool creates a new mock instance.
 func NewMockBackendPool(ctrl *gomock.Controller) *MockBackendPool {
 	mock := &MockBackendPool{ctrl: ctrl}
 	mock.recorder = &MockBackendPoolMockRecorder{mock}
 	return mock
 }
 
-// EXPECT returns an object that allows the caller to indicate expected use
+// EXPECT returns an object that allows the caller to indicate expected use.
 func (m *MockBackendPool) EXPECT() *MockBackendPoolMockRecorder {
 	return m.recorder
 }
 
-// EnsureHostsInPool mocks base method
-func (m *MockBackendPool) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "EnsureHostsInPool", service, nodes, backendPoolID, vmSetName, clusterName, lbName, backendPool)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// EnsureHostsInPool indicates an expected call of EnsureHostsInPool
-func (mr *MockBackendPoolMockRecorder) EnsureHostsInPool(service, nodes, backendPoolID, vmSetName, clusterName, lbName, backendPool interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureHostsInPool", reflect.TypeOf((*MockBackendPool)(nil).EnsureHostsInPool), service, nodes, backendPoolID, vmSetName, clusterName, lbName, backendPool)
-}
-
-// CleanupVMSetFromBackendPoolByCondition mocks base method
+// CleanupVMSetFromBackendPoolByCondition mocks base method.
 func (m *MockBackendPool) CleanupVMSetFromBackendPoolByCondition(slb *network.LoadBalancer, service *v1.Service, nodes []*v1.Node, clusterName string, shouldRemoveVMSetFromSLB func(string) bool) (*network.LoadBalancer, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "CleanupVMSetFromBackendPoolByCondition", slb, service, nodes, clusterName, shouldRemoveVMSetFromSLB)
@@ -71,29 +56,27 @@ func (m *MockBackendPool) CleanupVMSetFromBackendPoolByCondition(slb *network.Lo
 	return ret0, ret1
 }
 
-// CleanupVMSetFromBackendPoolByCondition indicates an expected call of CleanupVMSetFromBackendPoolByCondition
+// CleanupVMSetFromBackendPoolByCondition indicates an expected call of CleanupVMSetFromBackendPoolByCondition.
 func (mr *MockBackendPoolMockRecorder) CleanupVMSetFromBackendPoolByCondition(slb, service, nodes, clusterName, shouldRemoveVMSetFromSLB interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanupVMSetFromBackendPoolByCondition", reflect.TypeOf((*MockBackendPool)(nil).CleanupVMSetFromBackendPoolByCondition), slb, service, nodes, clusterName, shouldRemoveVMSetFromSLB)
 }
 
-// ReconcileBackendPools mocks base method
-func (m *MockBackendPool) ReconcileBackendPools(clusterName string, service *v1.Service, lb *network.LoadBalancer) (bool, bool, error) {
+// EnsureHostsInPool mocks base method.
+func (m *MockBackendPool) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, backendPoolID, vmSetName, clusterName, lbName string, backendPool network.BackendAddressPool) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ReconcileBackendPools", clusterName, service, lb)
-	ret0, _ := ret[0].(bool)
-	ret1, _ := ret[1].(bool)
-	ret2, _ := ret[2].(error)
-	return ret0, ret1, ret2
+	ret := m.ctrl.Call(m, "EnsureHostsInPool", service, nodes, backendPoolID, vmSetName, clusterName, lbName, backendPool)
+	ret0, _ := ret[0].(error)
+	return ret0
 }
 
-// ReconcileBackendPools indicates an expected call of ReconcileBackendPools
-func (mr *MockBackendPoolMockRecorder) ReconcileBackendPools(clusterName, service, lb interface{}) *gomock.Call {
+// EnsureHostsInPool indicates an expected call of EnsureHostsInPool.
+func (mr *MockBackendPoolMockRecorder) EnsureHostsInPool(service, nodes, backendPoolID, vmSetName, clusterName, lbName, backendPool interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileBackendPools", reflect.TypeOf((*MockBackendPool)(nil).ReconcileBackendPools), clusterName, service, lb)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureHostsInPool", reflect.TypeOf((*MockBackendPool)(nil).EnsureHostsInPool), service, nodes, backendPoolID, vmSetName, clusterName, lbName, backendPool)
 }
 
-// GetBackendPrivateIPs mocks base method
+// GetBackendPrivateIPs mocks base method.
 func (m *MockBackendPool) GetBackendPrivateIPs(clusterName string, service *v1.Service, lb *network.LoadBalancer) ([]string, []string) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetBackendPrivateIPs", clusterName, service, lb)
@@ -102,8 +85,25 @@ func (m *MockBackendPool) GetBackendPrivateIPs(clusterName string, service *v1.S
 	return ret0, ret1
 }
 
-// GetBackendPrivateIPs indicates an expected call of GetBackendPrivateIPs
+// GetBackendPrivateIPs indicates an expected call of GetBackendPrivateIPs.
 func (mr *MockBackendPoolMockRecorder) GetBackendPrivateIPs(clusterName, service, lb interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBackendPrivateIPs", reflect.TypeOf((*MockBackendPool)(nil).GetBackendPrivateIPs), clusterName, service, lb)
 }
+
+// ReconcileBackendPools mocks base method.
+func (m *MockBackendPool) ReconcileBackendPools(clusterName string, service *v1.Service, lb *network.LoadBalancer) (bool, bool, bool, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ReconcileBackendPools", clusterName, service, lb)
+	ret0, _ := ret[0].(bool)
+	ret1, _ := ret[1].(bool)
+	ret2, _ := ret[2].(bool)
+	ret3, _ := ret[3].(error)
+	return ret0, ret1, ret2, ret3
+}
+
+// ReconcileBackendPools indicates an expected call of ReconcileBackendPools.
+func (mr *MockBackendPoolMockRecorder) ReconcileBackendPools(clusterName, service, lb interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileBackendPools", reflect.TypeOf((*MockBackendPool)(nil).ReconcileBackendPools), clusterName, service, lb)
+}
diff --git a/pkg/provider/azure_mock_vmsets.go b/pkg/provider/azure_mock_vmsets.go
index f461cb821..6a64e259f 100644
--- a/pkg/provider/azure_mock_vmsets.go
+++ b/pkg/provider/azure_mock_vmsets.go
@@ -24,6 +24,7 @@ import (
 	network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
 	azure "github.com/Azure/go-autorest/autorest/azure"
 	gomock "github.com/golang/mock/gomock"
+
 	v1 "k8s.io/api/core/v1"
 	types "k8s.io/apimachinery/pkg/types"
 	cloud_provider "k8s.io/cloud-provider"
@@ -415,7 +416,7 @@ func (mr *MockVMSetMockRecorder) UpdateVM(ctx, nodeName interface{}) *gomock.Cal
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVM", reflect.TypeOf((*MockVMSet)(nil).UpdateVM), ctx, nodeName)
 }
 
-// UpdateVMAsync mocks base method
+// UpdateVMAsync mocks base method.
 func (m *MockVMSet) UpdateVMAsync(ctx context.Context, nodeName types.NodeName) (*azure.Future, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "UpdateVMAsync", ctx, nodeName)
@@ -424,7 +425,7 @@ func (m *MockVMSet) UpdateVMAsync(ctx context.Context, nodeName types.NodeName)
 	return ret0, ret1
 }
 
-// UpdateVMAsync indicates an expected call of UpdateVMAsync
+// UpdateVMAsync indicates an expected call of UpdateVMAsync.
 func (mr *MockVMSetMockRecorder) UpdateVMAsync(ctx, nodeName interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVMAsync", reflect.TypeOf((*MockVMSet)(nil).UpdateVMAsync), ctx, nodeName)
@@ -433,7 +434,7 @@ func (mr *MockVMSetMockRecorder) UpdateVMAsync(ctx, nodeName interface{}) *gomoc
 // WaitForUpdateResult mocks base method.
 func (m *MockVMSet) WaitForUpdateResult(ctx context.Context, future *azure.Future, nodeName types.NodeName, source string) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "WaitForUpdateResult", ctx, future, source)
+	ret := m.ctrl.Call(m, "WaitForUpdateResult", ctx, future, nodeName, source)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
@@ -441,5 +442,5 @@ func (m *MockVMSet) WaitForUpdateResult(ctx context.Context, future *azure.Futur
 // WaitForUpdateResult indicates an expected call of WaitForUpdateResult.
 func (mr *MockVMSetMockRecorder) WaitForUpdateResult(ctx, future, nodeName, source interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForUpdateResult", reflect.TypeOf((*MockVMSet)(nil).WaitForUpdateResult), ctx, future, source)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForUpdateResult", reflect.TypeOf((*MockVMSet)(nil).WaitForUpdateResult), ctx, future, nodeName, source)
 }
diff --git a/pkg/provider/azure_utils.go b/pkg/provider/azure_utils.go
index 7cd8fa1c8..3922659bf 100644
--- a/pkg/provider/azure_utils.go
+++ b/pkg/provider/azure_utils.go
@@ -495,3 +495,38 @@ func getResourceIDPrefix(id string) string {
 	}
 	return id[:idx]
 }
+
+func getLBNameFromBackendPoolID(backendPoolID string) (string, error) {
+	matches := backendPoolIDRE.FindStringSubmatch(backendPoolID)
+	if len(matches) != 2 {
+		return "", fmt.Errorf("backendPoolID %q is in wrong format", backendPoolID)
+	}
+
+	return matches[1], nil
+}
+
+func countNICsOnBackendPool(backendPool network.BackendAddressPool) int {
+	if backendPool.BackendAddressPoolPropertiesFormat == nil ||
+		backendPool.BackendIPConfigurations == nil {
+		return 0
+	}
+
+	return len(*backendPool.BackendIPConfigurations)
+}
+
+func countIPsOnBackendPool(backendPool network.BackendAddressPool) int {
+	if backendPool.BackendAddressPoolPropertiesFormat == nil ||
+		backendPool.LoadBalancerBackendAddresses == nil {
+		return 0
+	}
+
+	var ipsCount int
+	for _, loadBalancerBackendAddress := range *backendPool.LoadBalancerBackendAddresses {
+		if loadBalancerBackendAddress.LoadBalancerBackendAddressPropertiesFormat != nil &&
+			pointer.StringDeref(loadBalancerBackendAddress.IPAddress, "") != "" {
+			ipsCount++
+		}
+	}
+
+	return ipsCount
+}
diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go
index 8e22e283b..6dd2e48e9 100644
--- a/pkg/provider/azure_vmss.go
+++ b/pkg/provider/azure_vmss.go
@@ -1555,7 +1555,7 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromNode(nodeName string, backendPoo
 
 	// Find primary network interface configuration.
 	if vm.VirtualMachineScaleSetVMProperties.NetworkProfileConfiguration.NetworkInterfaceConfigurations == nil {
-		klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vm %s, "+
+		klog.V(4).Infof("ensureBackendPoolDeletedFromNode: cannot obtain the primary network interface configuration, of vm %s, "+
 			"probably because the vm's being deleted", nodeName)
 		return "", "", "", nil, nil
 	}
diff --git a/site/content/en/install/configs.md b/site/content/en/install/configs.md
index 0c57621fc..053f1857f 100644
--- a/site/content/en/install/configs.md
+++ b/site/content/en/install/configs.md
@@ -67,16 +67,16 @@ If more than one value is set, the order is `Managed Identity` > `Service Princi
 
 | Name                                                       | Description                                                                                                                                                                                                       | Remark                                                                                                                                |
 |------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
-| resourceGroup                                              | The name of the resource group that the cluster is deployed in                                                                                                                                                    ||
-| location                                                   | The location of the resource group that the cluster is deployed in                                                                                                                                                ||
-| vnetName                                                   | The name of the VNet that the cluster is deployed in                                                                                                                                                              ||
-| vnetResourceGroup                                          | The name of the resource group that the Vnet is deployed in                                                                                                                                                       ||
-| subnetName                                                 | The name of the subnet that the cluster is deployed in                                                                                                                                                            ||
-| securityGroupName                                          | The name of the security group attached to the cluster's subnet                                                                                                                                                   ||
-| securityGroupResourceGroup                                 | The name of the resource group that the security group is deployed in                                                                                                                                             ||
+| resourceGroup                                              | The name of the resource group that the cluster is deployed in                                                                                                                                                    |                                                                                                                                       |
+| location                                                   | The location of the resource group that the cluster is deployed in                                                                                                                                                |                                                                                                                                       |
+| vnetName                                                   | The name of the VNet that the cluster is deployed in                                                                                                                                                              |                                                                                                                                       |
+| vnetResourceGroup                                          | The name of the resource group that the Vnet is deployed in                                                                                                                                                       |                                                                                                                                       |
+| subnetName                                                 | The name of the subnet that the cluster is deployed in                                                                                                                                                            |                                                                                                                                       |
+| securityGroupName                                          | The name of the security group attached to the cluster's subnet                                                                                                                                                   |                                                                                                                                       |
+| securityGroupResourceGroup                                 | The name of the resource group that the security group is deployed in                                                                                                                                             |                                                                                                                                       |
 | routeTableName                                             | The name of the route table attached to the subnet that the cluster is deployed in                                                                                                                                | Optional in 1.6                                                                                                                       |
 | primaryAvailabilitySetName[*](#primaryavailabilitysetname) | The name of the availability set that should be used as the load balancer backend                                                                                                                                 | Optional                                                                                                                              |
-| vmType                                                     | The type of azure nodes. Candidate values are: `vmss`, `vmssflex` and `standard`                                                                                                                                              | Optional, default to `standard`                                                                                                       |
+| vmType                                                     | The type of azure nodes. Candidate values are: `vmss`, `vmssflex` and `standard`                                                                                                                                  | Optional, default to `standard`                                                                                                       |
 | primaryScaleSetName[*](#primaryscalesetname)               | The name of the scale set that should be used as the load balancer backend                                                                                                                                        | Optional                                                                                                                              |
 | cloudProviderBackoff                                       | Enable exponential backoff to manage resource request retries                                                                                                                                                     | Boolean value, default to false                                                                                                       |
 | cloudProviderBackoffRetries                                | Backoff retry limit                                                                                                                                                                                               | Integer value, valid if `cloudProviderBackoff` is true                                                                                |
@@ -97,7 +97,7 @@ If more than one value is set, the order is `Managed Identity` > `Service Princi
 | routeTableResourceGroup                                    | The resource group name for routeTable                                                                                                                                                                            | Default same as resourceGroup and available since v1.15.0                                                                             |
 | loadBalancerName                                           | Working together with loadBalancerResourceGroup to determine the LB name in a different resource group                                                                                                            | Since v1.18.0, default is cluster name setting on kube-controller-manager                                                             |
 | loadBalancerResourceGroup                                  | The load balancer resource group name, which is different from node resource group                                                                                                                                | Since v1.18.0, default is same as resourceGroup                                                                                       |
-| disableAvailabilitySetNodes                                | Disable supporting for AvailabilitySet virtual machines in vmss cluster. It should only be used when vmType is "vmss" and all the nodes (including master) are VMSS Uniform virtual machines                      | Since v1.18.0, default is false      
+| disableAvailabilitySetNodes                                | Disable supporting for AvailabilitySet virtual machines in vmss cluster. It should only be used when vmType is "vmss" and all the nodes (including master) are VMSS Uniform virtual machines                      | Since v1.18.0, default is false                                                                                                       |
 | enableVmssFlexNodes                                        | Enable supporting for VMSS Flex virtual machines in vmss cluster. It should only be used when vmType is "vmss"                                                                                                    | Since v1.26.0, default is false                                                                                                       |
 | availabilitySetNodesCacheTTLInSeconds                      | Cache TTL in seconds for availabilitySet Nodes                                                                                                                                                                    | Since v1.18.0, default is 900                                                                                                         |
 | vmssCacheTTLInSeconds                                      | Cache TTL in seconds for VMSS                                                                                                                                                                                     | Since v1.18.0, default is 600                                                                                                         |
@@ -113,6 +113,7 @@ If more than one value is set, the order is `Managed Identity` > `Service Princi
 | enableMultipleStandardLoadBalancers                        | Enable multiple standard Load Balancers per cluster.                                                                                                                                                              | Optional. Supported since v1.20.0                                                                                                     |
 | loadBalancerBackendPoolConfigurationType                   | The type of the Load Balancer backend pool. Supported values are `nodeIPConfiguration` (default) and `nodeIP`                                                                                                     | Optional. Supported since v1.23.0                                                                                                     |
 | putVMSSVMBatchSize                                         | The number of requests the client sends concurrently in a batch when putting the VMSS VMs. Anything smaller than or equal to 0 means to update VMSS VMs one by one in sequence.                                   | Optional. Supported since v1.24.0.                                                                                                    |
+| enableMigrateToIPBasedBackendPoolAPI                       | Use the migration API to migrate from NIC-based to IP-based Load Balancer backend pools without downtime.                                                                                                         | Optional. Supported since v1.24.0.                                                                                                    |
 
 ### primaryAvailabilitySetName
 
diff --git a/site/content/en/topics/loadbalancer.md b/site/content/en/topics/loadbalancer.md
index 1804db3ac..a7c5c24d4 100644
--- a/site/content/en/topics/loadbalancer.md
+++ b/site/content/en/topics/loadbalancer.md
@@ -290,6 +290,16 @@ The backend pool type can be configured by specifying `loadBalancerBackendPoolCo
 2. `nodeIP`. In this case we attach nodes to the LB by calling the LB API to add the node private IP addresses to the LB backend pool.
 3. `podIP` (not supported yet). In this case we do not attach nodes to the LB. Instead we directly adding pod IPs to the LB backend pool.
 
+To migrate from one backend pool type to another, just change the value of `loadBalancerBackendPoolConfigurationType` and re-apply the cloud configuration file. There will be downtime during the migration process.
+
+### Migration API from `nodeIPConfiguration` to `nodeIP`
+
+> This feature is supported since v1.24.0
+
+The migration from `nodeIPConfiguration` to `nodeIP` can be done without downtime by configuring `"enableMigrateToIPBasedBackendPoolAPI": true` in the cloud configuration file.
+
+```bash
+
 ## Load balancer limits
 
 The limits of the load balancer related resources are listed below:


Test patch:
diff --git a/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient_test.go b/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient_test.go
index 9bc538f6e..c33331a2e 100644
--- a/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient_test.go
+++ b/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient_test.go
@@ -39,8 +39,9 @@ import (
 )
 
 const (
-	testResourceID     = "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb1"
-	testResourcePrefix = "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/loadBalancers"
+	testResourceID            = "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb1"
+	testBackendPoolResourceID = "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb1/backendAddressPools/lb1"
+	testResourcePrefix        = "/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/loadBalancers"
 )
 
 func TestNew(t *testing.T) {
@@ -328,6 +329,67 @@ func TestDelete(t *testing.T) {
 	}
 }
 
+func TestGetLBBackendPoolInternalError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusInternalServerError,
+		Body:       io.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResourceWithExpandQuery(gomock.Any(), testBackendPoolResourceID, "").Return(response, nil)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any())
+
+	lbClient := getTestLoadBalancerClient(armClient)
+	expected := network.BackendAddressPool{Response: autorest.Response{}}
+	result, rerr := lbClient.GetLBBackendPool(context.TODO(), "rg", "lb1", "lb1", "")
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode)
+}
+
+func TestGetLBBackendPoolThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       io.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResourceWithExpandQuery(gomock.Any(), testBackendPoolResourceID, "").Return(response, throttleErr)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any())
+
+	lbClient := getTestLoadBalancerClient(armClient)
+	result, rerr := lbClient.GetLBBackendPool(context.TODO(), "rg", "lb1", "lb1", "")
+	assert.Empty(t, result)
+	assert.Equal(t, throttleErr, rerr)
+}
+
+func TestMigrateToIpBasedBackendPools(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(bytes.NewReader([]byte(""))),
+	}
+	armClient.EXPECT().PostResource(gomock.Any(), testResourceID, "migrateToIpBased", backendPoolsToBeMigrated{BackendPoolNames: []string{"lb1"}}, gomock.Any()).Return(response, nil)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any())
+
+	lbClient := getTestLoadBalancerClient(armClient)
+	rerr := lbClient.MigrateToIPBasedBackendPool(context.TODO(), "rg", "lb1", []string{"lb1"})
+	assert.Nil(t, rerr)
+}
+
 func getTestLoadBalancer(name string) network.LoadBalancer {
 	return network.LoadBalancer{
 		ID:       pointer.String(fmt.Sprintf("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/%s", name)),
diff --git a/pkg/provider/azure_backoff_test.go b/pkg/provider/azure_backoff_test.go
index 18b8c3a6e..620d4b616 100644
--- a/pkg/provider/azure_backoff_test.go
+++ b/pkg/provider/azure_backoff_test.go
@@ -666,3 +666,83 @@ func TestDeleteLBBackendPool(t *testing.T) {
 		assert.Equal(t, tc.expectedErr, err != nil)
 	}
 }
+
+func TestMigrateToIPBasedBackendPoolAndWaitForCompletion(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	for _, tc := range []struct {
+		desc                  string
+		migrationError        *retry.Error
+		backendPool           network.BackendAddressPool
+		backendPoolAfterRetry *network.BackendAddressPool
+		getBackendPoolError   *retry.Error
+		expectedError         error
+	}{
+		{
+			desc:           "MigrateToIPBasedBackendPoolAndWaitForCompletion should return the error if the migration fails",
+			migrationError: retry.NewError(false, errors.New("error")),
+			expectedError:  retry.NewError(false, errors.New("error")).Error(),
+		},
+		{
+			desc:                "MigrateToIPBasedBackendPoolAndWaitForCompletion should return the error if failed to get the backend pool",
+			getBackendPoolError: retry.NewError(false, errors.New("error")),
+			expectedError:       retry.NewError(false, errors.New("error")).Error(),
+		},
+		{
+			desc: "MigrateToIPBasedBackendPoolAndWaitForCompletion should retry if the number IPs on the backend pool is not expected",
+			backendPool: network.BackendAddressPool{
+				Name: pointer.String(testClusterName),
+				BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
+					LoadBalancerBackendAddresses: &[]network.LoadBalancerBackendAddress{
+						{
+							LoadBalancerBackendAddressPropertiesFormat: &network.LoadBalancerBackendAddressPropertiesFormat{
+								IPAddress: pointer.String("1.2.3.4"),
+							},
+						},
+					},
+				},
+			},
+			backendPoolAfterRetry: &network.BackendAddressPool{
+				Name: pointer.String(testClusterName),
+				BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
+					LoadBalancerBackendAddresses: &[]network.LoadBalancerBackendAddress{
+						{
+							LoadBalancerBackendAddressPropertiesFormat: &network.LoadBalancerBackendAddressPropertiesFormat{
+								IPAddress: pointer.String("1.2.3.4"),
+							},
+						},
+						{
+							LoadBalancerBackendAddressPropertiesFormat: &network.LoadBalancerBackendAddressPropertiesFormat{
+								IPAddress: pointer.String("2.3.4.5"),
+							},
+						},
+					},
+				},
+			},
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			az := GetTestCloud(ctrl)
+			lbClient := mockloadbalancerclient.NewMockInterface(ctrl)
+			lbClient.EXPECT().MigrateToIPBasedBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.migrationError)
+			if tc.migrationError == nil {
+				lbClient.EXPECT().GetLBBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.backendPool, tc.getBackendPoolError)
+			}
+			if tc.backendPoolAfterRetry != nil {
+				lbClient.EXPECT().GetLBBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(*tc.backendPoolAfterRetry, nil)
+			}
+			az.LoadBalancerClient = lbClient
+
+			lbName := testClusterName
+			backendPoolNames := []string{testClusterName}
+			nicsCountMap := map[string]int{testClusterName: 2}
+			err := az.MigrateToIPBasedBackendPoolAndWaitForCompletion(lbName, backendPoolNames, nicsCountMap)
+			if tc.expectedError != nil {
+				assert.EqualError(t, err, tc.expectedError.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
diff --git a/pkg/provider/azure_loadbalancer_backendpool_test.go b/pkg/provider/azure_loadbalancer_backendpool_test.go
index 812c6ea55..e091aab77 100644
--- a/pkg/provider/azure_loadbalancer_backendpool_test.go
+++ b/pkg/provider/azure_loadbalancer_backendpool_test.go
@@ -381,7 +381,7 @@ func TestReconcileBackendPoolsNodeIPConfig(t *testing.T) {
 
 	bc := newBackendPoolTypeNodeIPConfig(az)
 	svc := getTestService("test", v1.ProtocolTCP, nil, false, 80)
-	_, _, err := bc.ReconcileBackendPools(testClusterName, &svc, &lb)
+	_, _, _, err := bc.ReconcileBackendPools(testClusterName, &svc, &lb)
 	assert.NoError(t, err)
 
 	lb = network.LoadBalancer{
@@ -391,9 +391,10 @@ func TestReconcileBackendPoolsNodeIPConfig(t *testing.T) {
 	az = GetTestCloud(ctrl)
 	az.PreConfiguredBackendPoolLoadBalancerTypes = consts.PreConfiguredBackendPoolLoadBalancerTypesAll
 	bc = newBackendPoolTypeNodeIPConfig(az)
-	preConfigured, changed, err := bc.ReconcileBackendPools(testClusterName, &svc, &lb)
+	preConfigured, changed, shouldRefresh, err := bc.ReconcileBackendPools(testClusterName, &svc, &lb)
 	assert.NoError(t, err)
 	assert.False(t, preConfigured)
+	assert.False(t, shouldRefresh)
 	assert.True(t, changed)
 }
 
@@ -421,11 +422,11 @@ func TestReconcileBackendPoolsNodeIPConfigRemoveIPConfig(t *testing.T) {
 
 	bc := newBackendPoolTypeNodeIPConfig(az)
 	svc := getTestService("test", v1.ProtocolTCP, nil, false, 80)
-	_, _, err := bc.ReconcileBackendPools(testClusterName, &svc, &lb)
+	_, _, _, err := bc.ReconcileBackendPools(testClusterName, &svc, &lb)
 	assert.NoError(t, err)
 
 	mockVMSet.EXPECT().GetNodeNameByIPConfigurationID("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1").Return("k8s-agentpool1-00000000", "", errors.New("error"))
-	_, _, err = bc.ReconcileBackendPools(testClusterName, &svc, &lb)
+	_, _, _, err = bc.ReconcileBackendPools(testClusterName, &svc, &lb)
 	assert.Equal(t, "error", err.Error())
 }
 
@@ -449,7 +450,7 @@ func TestReconcileBackendPoolsNodeIPConfigPreConfigured(t *testing.T) {
 
 	svc := getTestService("test", v1.ProtocolTCP, nil, false, 80)
 	bc := newBackendPoolTypeNodeIPConfig(az)
-	preConfigured, changed, err := bc.ReconcileBackendPools(testClusterName, &svc, &lb)
+	preConfigured, changed, _, err := bc.ReconcileBackendPools(testClusterName, &svc, &lb)
 	assert.True(t, preConfigured)
 	assert.False(t, changed)
 	assert.NoError(t, err)
@@ -469,13 +470,14 @@ func TestReconcileBackendPoolsNodeIPToIPConfig(t *testing.T) {
 
 	bc := newBackendPoolTypeNodeIPConfig(az)
 	svc := getTestService("test", v1.ProtocolTCP, nil, false, 80)
-	_, _, err := bc.ReconcileBackendPools(testClusterName, &svc, lb)
+	_, _, _, err := bc.ReconcileBackendPools(testClusterName, &svc, lb)
 	assert.Contains(t, err.Error(), "create or update LB backend pool error")
 
 	lb = buildLBWithVMIPs(testClusterName, []string{"10.0.0.1", "10.0.0.2"})
 	mockLBClient.EXPECT().CreateOrUpdateBackendPools(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
-	_, _, err = bc.ReconcileBackendPools(testClusterName, &svc, lb)
+	_, _, shouldRefresh, err := bc.ReconcileBackendPools(testClusterName, &svc, lb)
 	assert.NoError(t, err)
+	assert.False(t, shouldRefresh)
 	assert.Empty(t, (*lb.BackendAddressPools)[0].LoadBalancerBackendAddresses)
 }
 
@@ -537,14 +539,14 @@ func TestReconcileBackendPoolsNodeIP(t *testing.T) {
 
 	lbClient := mockloadbalancerclient.NewMockInterface(ctrl)
 	lbClient.EXPECT().CreateOrUpdateBackendPools(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), bp, gomock.Any()).Return(nil)
-	lbClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(network.LoadBalancer{}, nil)
 	az.LoadBalancerClient = lbClient
 
 	bi := newBackendPoolTypeNodeIP(az)
 
 	service := getTestService("test", v1.ProtocolTCP, nil, false, 80)
 
-	_, _, err := bi.ReconcileBackendPools("kubernetes", &service, lb)
+	_, _, shouldRefresh, err := bi.ReconcileBackendPools("kubernetes", &service, lb)
+	assert.True(t, shouldRefresh)
 	assert.NoError(t, err)
 
 	lb = &network.LoadBalancer{
@@ -554,9 +556,10 @@ func TestReconcileBackendPoolsNodeIP(t *testing.T) {
 	az = GetTestCloud(ctrl)
 	az.PreConfiguredBackendPoolLoadBalancerTypes = consts.PreConfiguredBackendPoolLoadBalancerTypesAll
 	bi = newBackendPoolTypeNodeIP(az)
-	preConfigured, changed, err := bi.ReconcileBackendPools(testClusterName, &service, lb)
+	preConfigured, changed, shouldRefresh, err := bi.ReconcileBackendPools(testClusterName, &service, lb)
 	assert.NoError(t, err)
 	assert.False(t, preConfigured)
+	assert.False(t, shouldRefresh)
 	assert.True(t, changed)
 }
 
@@ -570,18 +573,15 @@ func TestReconcileBackendPoolsNodeIPEmptyPool(t *testing.T) {
 	mockVMSet.EXPECT().EnsureBackendPoolDeleted(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil)
 	mockVMSet.EXPECT().GetPrimaryVMSetName().Return("k8s-agentpool1-00000000")
 
-	lbClient := mockloadbalancerclient.NewMockInterface(ctrl)
-	lbClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(network.LoadBalancer{}, nil)
-
 	az := GetTestCloud(ctrl)
 	az.LoadBalancerBackendPoolConfigurationType = consts.LoadBalancerBackendPoolConfigurationTypeNodeIP
 	az.VMSet = mockVMSet
-	az.LoadBalancerClient = lbClient
 	bi := newBackendPoolTypeNodeIP(az)
 
 	service := getTestService("test", v1.ProtocolTCP, nil, false, 80)
 
-	_, _, err := bi.ReconcileBackendPools("kubernetes", &service, lb)
+	_, _, shouldRefresh, err := bi.ReconcileBackendPools("kubernetes", &service, lb)
+	assert.True(t, shouldRefresh)
 	assert.NoError(t, err)
 }
 
@@ -601,9 +601,10 @@ func TestReconcileBackendPoolsNodeIPPreConfigured(t *testing.T) {
 
 	service := getTestService("test", v1.ProtocolTCP, nil, false, 80)
 	bi := newBackendPoolTypeNodeIP(az)
-	preConfigured, changed, err := bi.ReconcileBackendPools("kubernetes", &service, lb)
+	preConfigured, changed, shouldRefresh, err := bi.ReconcileBackendPools("kubernetes", &service, lb)
 	assert.True(t, preConfigured)
 	assert.False(t, changed)
+	assert.False(t, shouldRefresh)
 	assert.NoError(t, err)
 }
 
@@ -623,7 +624,7 @@ func TestReconcileBackendPoolsNodeIPConfigToIP(t *testing.T) {
 	az.VMSet = mockVMSet
 	bi := newBackendPoolTypeNodeIP(az)
 	svc := getTestService("test", v1.ProtocolTCP, nil, false, 80)
-	_, _, err := bi.ReconcileBackendPools(testClusterName, &svc, &lb)
+	_, _, _, err := bi.ReconcileBackendPools(testClusterName, &svc, &lb)
 	assert.Contains(t, err.Error(), "delete LB backend pool error")
 
 	lb = buildDefaultTestLB(testClusterName, []string{
@@ -631,11 +632,47 @@ func TestReconcileBackendPoolsNodeIPConfigToIP(t *testing.T) {
 		"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool2-00000000-nic-1/ipConfigurations/ipconfig1",
 	})
 	mockVMSet.EXPECT().EnsureBackendPoolDeleted(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil)
-	_, _, err = bi.ReconcileBackendPools(testClusterName, &svc, &lb)
+	_, _, shouldRefresh, err := bi.ReconcileBackendPools(testClusterName, &svc, &lb)
 	assert.NoError(t, err)
+	assert.True(t, shouldRefresh)
 	assert.Empty(t, (*lb.BackendAddressPools)[0].LoadBalancerBackendAddresses)
 }
 
+func TestReconcileBackendPoolsNodeIPConfigToIPWithMigrationAPI(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	lb := buildDefaultTestLB(testClusterName, []string{
+		"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool1-00000000-nic-1/ipConfigurations/ipconfig1",
+		"/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/k8s-agentpool2-00000000-nic-1/ipConfigurations/ipconfig1",
+	})
+
+	mockVMSet := NewMockVMSet(ctrl)
+	mockVMSet.EXPECT().EnsureBackendPoolDeleted(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil)
+	mockVMSet.EXPECT().GetPrimaryVMSetName().Return("k8s-agentpool1-00000000").AnyTimes()
+
+	mockLBClient := mockloadbalancerclient.NewMockInterface(ctrl)
+	mockLBClient.EXPECT().MigrateToIPBasedBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(retry.NewError(false, errors.New("error")))
+
+	az := GetTestCloud(ctrl)
+	az.VMSet = mockVMSet
+	az.LoadBalancerClient = mockLBClient
+	az.EnableMigrateToIPBasedBackendPoolAPI = true
+
+	bi := newBackendPoolTypeNodeIP(az)
+	svc := getTestService("test", v1.ProtocolTCP, nil, false, 80)
+	_, _, _, err := bi.ReconcileBackendPools(testClusterName, &svc, &lb)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "error")
+
+	mockLBClient.EXPECT().MigrateToIPBasedBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
+	bps := buildLBWithVMIPs(testClusterName, []string{"1.2.3.4", "2.3.4.5"}).BackendAddressPools
+	mockLBClient.EXPECT().GetLBBackendPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return((*bps)[0], nil)
+	_, _, shouldRefresh, err := bi.ReconcileBackendPools(testClusterName, &svc, &lb)
+	assert.NoError(t, err)
+	assert.True(t, shouldRefresh)
+}
+
 func buildTestLoadBalancerBackendPoolWithIPs(name string, ips []string) network.BackendAddressPool {
 	backendPool := network.BackendAddressPool{
 		Name: &name,
diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go
index 49c14a817..12704f0ef 100644
--- a/pkg/provider/azure_loadbalancer_test.go
+++ b/pkg/provider/azure_loadbalancer_test.go
@@ -857,7 +857,7 @@ func TestEnsureLoadBalancerDeleted(t *testing.T) {
 	defer ctrl.Finish()
 	az := GetTestCloud(ctrl)
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 	mockLBBackendPool.EXPECT().GetBackendPrivateIPs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
 
@@ -3384,17 +3384,19 @@ func TestReconcileLoadBalancer(t *testing.T) {
 	}
 
 	testCases := []struct {
-		desc                      string
-		service                   v1.Service
-		loadBalancerSku           string
-		preConfigLBType           string
-		loadBalancerResourceGroup string
-		disableOutboundSnat       *bool
-		wantLb                    bool
-		existingLB                network.LoadBalancer
-		expectedLB                network.LoadBalancer
-		expectLBUpdate            bool
-		expectedError             error
+		desc                                      string
+		service                                   v1.Service
+		loadBalancerSku                           string
+		preConfigLBType                           string
+		loadBalancerResourceGroup                 string
+		disableOutboundSnat                       *bool
+		wantLb                                    bool
+		shouldRefreshLBAfterReconcileBackendPools bool
+		existingLB                                network.LoadBalancer
+		expectedLB                                network.LoadBalancer
+		expectLBUpdate                            bool
+		expectedGetLBError                        *retry.Error
+		expectedError                             error
 	}{
 		{
 			desc: "reconcileLoadBalancer shall return the lb deeply equal to the existingLB if there's no " +
@@ -3490,6 +3492,25 @@ func TestReconcileLoadBalancer(t *testing.T) {
 			expectLBUpdate:            true,
 			expectedError:             nil,
 		},
+		{
+			desc:            "reconcileLoadBalancer should refresh the LB after reconciling backend pools if needed",
+			loadBalancerSku: "basic",
+			service:         service1,
+			existingLB:      basicLb1,
+			wantLb:          true,
+			shouldRefreshLBAfterReconcileBackendPools: true,
+			expectedLB:    basicLb1,
+			expectedError: nil,
+		},
+		{
+			desc:       "reconcileLoadBalancer should return the error if failed to get the LB after reconcileBackendPools",
+			service:    service1,
+			existingLB: basicLb1,
+			wantLb:     true,
+			shouldRefreshLBAfterReconcileBackendPools: true,
+			expectedGetLBError:                        retry.NewError(false, errors.New("error")),
+			expectedError:                             fmt.Errorf("reconcileLoadBalancer for service (default/service1): failed to get load balancer testCluster: %w", retry.NewError(false, errors.New("error")).Error()),
+		},
 	}
 
 	for _, test := range testCases {
@@ -3518,7 +3539,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
 
 			mockLBsClient := mockloadbalancerclient.NewMockInterface(ctrl)
 			mockLBsClient.EXPECT().List(gomock.Any(), az.getLoadBalancerResourceGroup()).Return([]network.LoadBalancer{test.existingLB}, nil)
-			mockLBsClient.EXPECT().Get(gomock.Any(), az.getLoadBalancerResourceGroup(), *test.existingLB.Name, gomock.Any()).Return(test.existingLB, nil).AnyTimes()
+			mockLBsClient.EXPECT().Get(gomock.Any(), az.getLoadBalancerResourceGroup(), *test.existingLB.Name, gomock.Any()).Return(test.existingLB, test.expectedGetLBError).AnyTimes()
 			expectLBUpdateCount := 1
 			if test.expectLBUpdate {
 				expectLBUpdateCount++
@@ -3530,7 +3551,10 @@ func TestReconcileLoadBalancer(t *testing.T) {
 			assert.NoError(t, err.Error())
 
 			mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-			mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+			if test.shouldRefreshLBAfterReconcileBackendPools {
+				mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, true, nil)
+			}
+			mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 			mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 			lb, rerr := az.reconcileLoadBalancer("testCluster", &service, clusterResources.nodes, test.wantLb)
diff --git a/pkg/provider/azure_test.go b/pkg/provider/azure_test.go
index 849ceeed4..88ae74b37 100644
--- a/pkg/provider/azure_test.go
+++ b/pkg/provider/azure_test.go
@@ -111,7 +111,7 @@ func TestAddPort(t *testing.T) {
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, false)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, err := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true /* wantLb */)
@@ -333,7 +333,7 @@ func testLoadBalancerServiceDefaultModeSelection(t *testing.T, isInternal bool)
 	setMockEnv(az, ctrl, expectedInterfaces, expectedVirtualMachines, serviceCount)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 	mockLBBackendPool.EXPECT().GetBackendPrivateIPs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
 
@@ -402,7 +402,7 @@ func testLoadBalancerServiceAutoModeSelection(t *testing.T, isInternal bool) {
 	expectedLBs := make([]network.LoadBalancer, 0)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 	mockLBBackendPool.EXPECT().GetBackendPrivateIPs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
 
@@ -483,7 +483,7 @@ func testLoadBalancerServicesSpecifiedSelection(t *testing.T, isInternal bool) {
 	expectedLBs := make([]network.LoadBalancer, 0)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 	mockLBBackendPool.EXPECT().GetBackendPrivateIPs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
 
@@ -542,7 +542,7 @@ func testLoadBalancerMaxRulesServices(t *testing.T, isInternal bool) {
 	expectedLBs := make([]network.LoadBalancer, 0)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 	mockLBBackendPool.EXPECT().GetBackendPrivateIPs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
 
@@ -627,7 +627,7 @@ func testLoadBalancerServiceAutoModeDeleteSelection(t *testing.T, isInternal boo
 	expectedLBs := make([]network.LoadBalancer, 0)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 	mockLBBackendPool.EXPECT().GetBackendPrivateIPs(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
 
@@ -721,7 +721,7 @@ func TestReconcileLoadBalancerAddServiceOnInternalSubnet(t *testing.T) {
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, true)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, err := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true /* wantLb */)
@@ -796,7 +796,7 @@ func TestReconcileLoadBalancerAddServicesOnMultipleSubnets(t *testing.T) {
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, false)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	// svc1 is using LB without "-internal" suffix
@@ -848,7 +848,7 @@ func TestReconcileLoadBalancerEditServiceSubnet(t *testing.T) {
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, true)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	expectedPLS := make([]network.PrivateLinkService, 0)
@@ -897,7 +897,7 @@ func TestReconcileLoadBalancerNodeHealth(t *testing.T) {
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, false)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, err := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true /* wantLb */)
@@ -928,7 +928,7 @@ func TestReconcileLoadBalancerRemoveService(t *testing.T) {
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, false)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	_, err := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true /* wantLb */)
@@ -972,7 +972,7 @@ func TestReconcileLoadBalancerRemoveAllPortsRemovesFrontendConfig(t *testing.T)
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, false)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, err := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true /* wantLb */)
@@ -1014,7 +1014,7 @@ func TestReconcileLoadBalancerRemovesPort(t *testing.T) {
 	setMockEnv(az, ctrl, expectedInterfaces, expectedVirtualMachines, 1)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	expectedLBs := make([]network.LoadBalancer, 0)
@@ -1052,7 +1052,7 @@ func TestReconcileLoadBalancerMultipleServices(t *testing.T) {
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, false)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	expectedPLS := make([]network.PrivateLinkService, 0)
@@ -1118,7 +1118,7 @@ func TestServiceDefaultsToNoSessionPersistence(t *testing.T) {
 	mockPIPsClient.EXPECT().List(gomock.Any(), az.ResourceGroup).Return([]network.PublicIPAddress{expectedPIP}, nil).AnyTimes()
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, err := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true /* wantLb */)
@@ -1177,7 +1177,7 @@ func TestServiceRespectsNoSessionAffinity(t *testing.T) {
 	mockPLSClient.EXPECT().List(gomock.Any(), az.Config.ResourceGroup).Return(expectedPLS, nil).MinTimes(1).MaxTimes(1)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, err := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true /* wantLb */)
@@ -1238,7 +1238,7 @@ func TestServiceRespectsClientIPSessionAffinity(t *testing.T) {
 	mockPLSClient.EXPECT().List(gomock.Any(), az.Config.ResourceGroup).Return(expectedPLS, nil).MinTimes(1).MaxTimes(1)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, err := az.reconcileLoadBalancer(testClusterName, &svc, clusterResources.nodes, true /* wantLb */)
@@ -1270,7 +1270,7 @@ func TestReconcileSecurityGroupNewServiceAddsPort(t *testing.T) {
 	expectedLBs := make([]network.LoadBalancer, 0)
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, false)
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 	lb, _ := az.reconcileLoadBalancer(testClusterName, &svc1, clusterResources.nodes, true)
 	lbStatus, _, _ := az.getServiceLoadBalancerStatus(&svc1, lb)
@@ -1298,7 +1298,7 @@ func TestReconcileSecurityGroupNewInternalServiceAddsPort(t *testing.T) {
 	setMockLBs(az, ctrl, &expectedLBs, "service", 1, 1, true)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, _ := az.reconcileLoadBalancer(testClusterName, &svc1, clusterResources.nodes, true)
@@ -1333,7 +1333,7 @@ func TestReconcileSecurityGroupRemoveService(t *testing.T) {
 			"Standard"), nil).AnyTimes()
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, _ := az.reconcileLoadBalancer(testClusterName, &service1, clusterResources.nodes, true)
@@ -1362,7 +1362,7 @@ func TestReconcileSecurityGroupRemoveServiceRemovesPort(t *testing.T) {
 	setMockEnv(az, ctrl, expectedInterfaces, expectedVirtualMachines, 1)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	getTestSecurityGroup(az, svc)
@@ -1402,7 +1402,7 @@ func TestReconcileSecurityWithSourceRanges(t *testing.T) {
 	setMockEnv(az, ctrl, expectedInterfaces, expectedVirtualMachines, 1)
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	getTestSecurityGroup(az, svc)
@@ -1453,7 +1453,7 @@ func TestReconcileSecurityGroupEtagMismatch(t *testing.T) {
 			"Standard"), nil).AnyTimes()
 
 	mockLBBackendPool := az.LoadBalancerBackendPool.(*MockBackendPool)
-	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, nil).AnyTimes()
+	mockLBBackendPool.EXPECT().ReconcileBackendPools(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, false, false, nil).AnyTimes()
 	mockLBBackendPool.EXPECT().EnsureHostsInPool(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 
 	lb, _ := az.reconcileLoadBalancer(testClusterName, &svc1, clusterResources.nodes, true)