Repository: kubernetes-sigs/cloud-provider-azure
Pull request #953 (2021-12-24 04:39:48)

feat: support sending sync requests concurrently in batches when putting vmss vm
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?

<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

/kind feature

#### What this PR does / why we need it:

Introduce a configuration option `putVMSSVMBatchSize`. If set, the sync requests will be sent concurrently in batches when putting vmss vms.

#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
Introduce a configuration option `putVMSSVMBatchSize`. If set, the sync requests will be sent concurrently in batches when putting vmss vms.
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```



Patch:
diff --git a/README.md b/README.md
index a7eff63a5..2f1449b05 100644
--- a/README.md
+++ b/README.md
@@ -20,18 +20,17 @@ The latest version of azure-cloud-controller-manager and azure-cloud-node-manage
 
 Version matrix:
 
-| Kubernetes version | cloud-provider version | cloud-provider branch | Notes                                                                   |
-|--------------------|------------------------|-----------------------|-------------------------------------------------------------------------|
-| master             | N/A                    | master                |
-| v1.23.x            | v1.23.0                | release-1.23          | The release versions will match the k8s release versions since v1.23.0. |
-| v1.22.x            | v1.1.3                 | release-1.1           |
-| v1.21.x            | v1.0.7                 | release-1.0           |
-| v1.20.x            | v0.7.10                | release-0.7           |
-| v1.19.x            | v0.6.0                 | release-0.6           |
-| v1.18.x            | v0.5.1                 | release-0.5           |
-| v1.17.x            | v0.4.1                 | N/A                   |
-| v1.16.x            | v0.3.0                 | N/A                   |
-| v1.15.x            | v0.2.0                 | N/A                   |
+|Kubernetes version|cloud-provider version|cloud-provider branch|
+|------------------|----------------------|---------------------|
+| master           | N/A                  | master              |
+| v1.22.x          | v1.1.3               | release-1.1         |
+| v1.21.x          | v1.0.7               | release-1.0         |
+| v1.20.x          | v0.7.10              | release-0.7         |
+| v1.19.x          | v0.6.0               | release-0.6         |
+| v1.18.x          | v0.5.1               | release-0.5         |
+| v1.17.x          | v0.4.1               | N/A                 |
+| v1.16.x          | v0.3.0               | N/A                 |
+| v1.15.x          | v0.2.0               | N/A                 |
 
 ## Build
 
diff --git a/examples/out-of-tree/cloud-controller-manager.yaml b/examples/out-of-tree/cloud-controller-manager.yaml
index 536e7a42e..56f833985 100644
--- a/examples/out-of-tree/cloud-controller-manager.yaml
+++ b/examples/out-of-tree/cloud-controller-manager.yaml
@@ -146,7 +146,7 @@ spec:
       effect: NoSchedule
   containers:
     - name: cloud-controller-manager
-      image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.23.0
+      image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.1.3
       imagePullPolicy: IfNotPresent
       command: ["cloud-controller-manager"]
       args:
diff --git a/examples/out-of-tree/cloud-node-manager.yaml b/examples/out-of-tree/cloud-node-manager.yaml
index e4343bf25..8c012eedf 100644
--- a/examples/out-of-tree/cloud-node-manager.yaml
+++ b/examples/out-of-tree/cloud-node-manager.yaml
@@ -79,7 +79,7 @@ spec:
         effect: NoSchedule
       containers:
       - name: cloud-node-manager
-        image: mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.23.0
+        image: mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.1.3
         imagePullPolicy: IfNotPresent
         command:
         - cloud-node-manager
diff --git a/go.mod b/go.mod
index 904320b0e..4009febc5 100644
--- a/go.mod
+++ b/go.mod
@@ -18,13 +18,13 @@ require (
 	github.com/stretchr/testify v1.7.0
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
 	golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e
-	k8s.io/api v0.23.1
-	k8s.io/apimachinery v0.23.1
+	k8s.io/api v0.23.0
+	k8s.io/apimachinery v0.23.0
 	k8s.io/apiserver v0.23.0
-	k8s.io/client-go v0.23.1
+	k8s.io/client-go v0.23.0
 	k8s.io/cloud-provider v0.23.0
 	k8s.io/component-base v0.23.0
-	k8s.io/component-helpers v0.23.1
+	k8s.io/component-helpers v0.23.0
 	k8s.io/controller-manager v0.23.0
 	k8s.io/klog/v2 v2.30.0
 	k8s.io/kubelet v0.23.0
diff --git a/go.sum b/go.sum
index 2063eee20..0d98e1d49 100644
--- a/go.sum
+++ b/go.sum
@@ -383,9 +383,8 @@ k8s.io/cloud-provider v0.23.0 h1:9LATZJu57XanN7po4Xfj6jTLp44uhKiu5Xa3+3Cutz0=
 k8s.io/cloud-provider v0.23.0/go.mod h1:vY9zulPUCjwCg9TBMh5Qi42JUZUNecQNF7FojNELWws=
 k8s.io/component-base v0.23.0 h1:UAnyzjvVZ2ZR1lF35YwtNY6VMN94WtOnArcXBu34es8=
 k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI=
+k8s.io/component-helpers v0.23.0 h1:qNbqN10QTefiWcCOPkHL/0nn81sdKVv6ZgEXcSyot/U=
 k8s.io/component-helpers v0.23.0/go.mod h1:liXMh6FZS4qamKtMJQ7uLHnFe3tlC86RX5mJEk/aerg=
-k8s.io/component-helpers v0.23.1 h1:Xrtj0LwXUqYyTPvN2bOE2UcqURX+uSBmKX1koNGhVxI=
-k8s.io/component-helpers v0.23.1/go.mod h1:ZK24U+2oXnBPcas2KolLigVVN9g5zOzaHLkHiQMFGr0=
 k8s.io/controller-manager v0.23.0 h1:D41cuVvXADHZatEJOlZJc/QddxUSlkN6xqiHy++WKk8=
 k8s.io/controller-manager v0.23.0/go.mod h1:6/IKItSv6p9FY3mSbHgsOYmt4y+HDxiC5hEFg9rJVc8=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
diff --git a/pkg/azureclients/armclient/azure_armclient.go b/pkg/azureclients/armclient/azure_armclient.go
index f65e6721c..350128667 100644
--- a/pkg/azureclients/armclient/azure_armclient.go
+++ b/pkg/azureclients/armclient/azure_armclient.go
@@ -416,46 +416,7 @@ func (c *Client) PutResource(ctx context.Context, resourceID string, parameters
 	return c.PutResourceWithDecorators(ctx, resourceID, parameters, putDecorators)
 }
 
-// PutResources puts a list of resources from resources map[resourceID]parameters.
-// Those resources sync requests are sequential while async requests are concurrent. It's especially
-// useful when the ARM API doesn't support concurrent requests.
-func (c *Client) PutResources(ctx context.Context, resources map[string]interface{}) map[string]*PutResourcesResponse {
-	if len(resources) == 0 {
-		return nil
-	}
-
-	// Sequential sync requests.
-	futures := make(map[string]*azure.Future)
-	responses := make(map[string]*PutResourcesResponse)
-	for resourceID, parameters := range resources {
-		decorators := []autorest.PrepareDecorator{
-			autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
-			autorest.WithJSON(parameters),
-		}
-		request, err := c.PreparePutRequest(ctx, decorators...)
-		dumpRequest(request, 10)
-		if err != nil {
-			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.prepare", resourceID, err)
-			responses[resourceID] = &PutResourcesResponse{
-				Error: retry.NewError(false, err),
-			}
-			continue
-		}
-
-		future, resp, clientErr := c.SendAsync(ctx, request)
-		defer c.CloseResponse(ctx, resp)
-		if clientErr != nil {
-			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.send", resourceID, clientErr.Error())
-			responses[resourceID] = &PutResourcesResponse{
-				Error: clientErr,
-			}
-			continue
-		}
-
-		futures[resourceID] = future
-	}
-
-	// Concurrent async requests.
+func (c *Client) waitAsync(ctx context.Context, futures map[string]*azure.Future, previousResponses map[string]*PutResourcesResponse) {
 	wg := sync.WaitGroup{}
 	var responseLock sync.Mutex
 	for resourceID, future := range futures {
@@ -478,22 +439,111 @@ func (c *Client) PutResources(ctx context.Context, resources map[string]interfac
 				}
 
 				responseLock.Lock()
-				responses[resourceID] = &PutResourcesResponse{
+				previousResponses[resourceID] = &PutResourcesResponse{
 					Error: retriableErr,
 				}
 				responseLock.Unlock()
 				return
 			}
+		}(resourceID, future)
+	}
+	wg.Wait()
+}
+
+// PutResources puts a list of resources from resources map[resourceID]parameters.
+// Those resources sync requests are sequential while async requests are concurrent. It's especially
+// useful when the ARM API doesn't support concurrent requests.
+func (c *Client) PutResources(ctx context.Context, resources map[string]interface{}) map[string]*PutResourcesResponse {
+	if len(resources) == 0 {
+		return nil
+	}
 
-			responseLock.Lock()
+	// Sequential sync requests.
+	futures := make(map[string]*azure.Future)
+	responses := make(map[string]*PutResourcesResponse)
+	for resourceID, parameters := range resources {
+		future, rerr := c.PutResourceAsync(ctx, resourceID, parameters)
+		if rerr != nil {
 			responses[resourceID] = &PutResourcesResponse{
-				Response: response,
+				Error: rerr,
 			}
-			responseLock.Unlock()
-		}(resourceID, future)
+			continue
+		}
+		futures[resourceID] = future
 	}
 
-	wg.Wait()
+	c.waitAsync(ctx, futures, responses)
+
+	return responses
+}
+
+// PutResourcesInBatches is similar with PutResources, but it sends sync request concurrently in batches.
+func (c *Client) PutResourcesInBatches(ctx context.Context, resources map[string]interface{}, batchSize int) map[string]*PutResourcesResponse {
+	if len(resources) == 0 {
+		return nil
+	}
+
+	if batchSize <= 0 {
+		klog.V(4).Infof("PutResourcesInBatches: batch size %d, put resources in sequence", batchSize)
+		return c.PutResources(ctx, resources)
+	}
+
+	if batchSize > len(resources) {
+		klog.V(4).Infof("PutResourcesInBatches: batch size %d, but the number of the resources is %d", batchSize, resources)
+		batchSize = len(resources)
+	}
+	klog.V(4).Infof("PutResourcesInBatches: send sync requests in parallel with the batch size %d", batchSize)
+
+	// Convert map to slice because it is more straightforward to
+	// loop over slice in batches than map.
+	type resourcesMeta struct {
+		resourceID string
+		parameters interface{}
+	}
+	resourcesList := make([]resourcesMeta, 0)
+	for resourceID, parameters := range resources {
+		resourcesList = append(resourcesList, resourcesMeta{
+			resourceID: resourceID,
+			parameters: parameters,
+		})
+	}
+
+	// Concurrent sync requests in batches.
+	futures := make(map[string]*azure.Future)
+	responses := make(map[string]*PutResourcesResponse)
+	wg := sync.WaitGroup{}
+	var responseLock, futuresLock sync.Mutex
+	for i := 0; i < len(resourcesList); i += batchSize {
+		j := i + batchSize
+		if j > len(resourcesList) {
+			j = len(resourcesList)
+		}
+
+		for k := i; k < j; k++ {
+			wg.Add(1)
+			go func(resourceID string, parameters interface{}) {
+				defer wg.Done()
+				future, rerr := c.PutResourceAsync(ctx, resourceID, parameters)
+				if rerr != nil {
+					responseLock.Lock()
+					responses[resourceID] = &PutResourcesResponse{
+						Error: rerr,
+					}
+					responseLock.Unlock()
+					return
+				}
+
+				futuresLock.Lock()
+				futures[resourceID] = future
+				futuresLock.Unlock()
+			}(resourcesList[k].resourceID, resourcesList[k].parameters)
+		}
+		wg.Wait()
+	}
+
+	// Concurrent async requests.
+	c.waitAsync(ctx, futures, responses)
+
 	return responses
 }
 
diff --git a/pkg/azureclients/armclient/interface.go b/pkg/azureclients/armclient/interface.go
index 20a36bb9e..0970f3e2e 100644
--- a/pkg/azureclients/armclient/interface.go
+++ b/pkg/azureclients/armclient/interface.go
@@ -70,6 +70,9 @@ type Interface interface {
 	// useful when the ARM API doesn't support concurrent requests.
 	PutResources(ctx context.Context, resources map[string]interface{}) map[string]*PutResourcesResponse
 
+	// PutResourcesInBatches is similar with PutResources, but it sends sync request concurrently in batches.
+	PutResourcesInBatches(ctx context.Context, resources map[string]interface{}, batchSize int) map[string]*PutResourcesResponse
+
 	// PutResourceWithDecorators puts a resource with decorators by resource ID
 	PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error)
 
diff --git a/pkg/azureclients/armclient/mockarmclient/interface.go b/pkg/azureclients/armclient/mockarmclient/interface.go
index c5f0c084a..279086e8c 100644
--- a/pkg/azureclients/armclient/mockarmclient/interface.go
+++ b/pkg/azureclients/armclient/mockarmclient/interface.go
@@ -23,40 +23,39 @@ package mockarmclient
 
 import (
 	context "context"
-	http "net/http"
-	reflect "reflect"
-
 	autorest "github.com/Azure/go-autorest/autorest"
 	azure "github.com/Azure/go-autorest/autorest/azure"
 	gomock "github.com/golang/mock/gomock"
+	http "net/http"
+	reflect "reflect"
 	armclient "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient"
 	retry "sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
-// MockInterface is a mock of Interface interface.
+// MockInterface is a mock of Interface interface
 type MockInterface struct {
 	ctrl     *gomock.Controller
 	recorder *MockInterfaceMockRecorder
 }
 
-// MockInterfaceMockRecorder is the mock recorder for MockInterface.
+// MockInterfaceMockRecorder is the mock recorder for MockInterface
 type MockInterfaceMockRecorder struct {
 	mock *MockInterface
 }
 
-// NewMockInterface creates a new mock instance.
+// NewMockInterface creates a new mock instance
 func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
 	mock := &MockInterface{ctrl: ctrl}
 	mock.recorder = &MockInterfaceMockRecorder{mock}
 	return mock
 }
 
-// EXPECT returns an object that allows the caller to indicate expected use.
+// EXPECT returns an object that allows the caller to indicate expected use
 func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
-// Send mocks base method.
+// Send mocks base method
 func (m *MockInterface) Send(ctx context.Context, request *http.Request) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Send", ctx, request)
@@ -65,13 +64,13 @@ func (m *MockInterface) Send(ctx context.Context, request *http.Request) (*http.
 	return ret0, ret1
 }
 
-// Send indicates an expected call of Send.
+// Send indicates an expected call of Send
 func (mr *MockInterfaceMockRecorder) Send(ctx, request interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockInterface)(nil).Send), ctx, request)
 }
 
-// PreparePutRequest mocks base method.
+// PreparePutRequest mocks base method
 func (m *MockInterface) PreparePutRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{ctx}
@@ -84,14 +83,14 @@ func (m *MockInterface) PreparePutRequest(ctx context.Context, decorators ...aut
 	return ret0, ret1
 }
 
-// PreparePutRequest indicates an expected call of PreparePutRequest.
+// PreparePutRequest indicates an expected call of PreparePutRequest
 func (mr *MockInterfaceMockRecorder) PreparePutRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{ctx}, decorators...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PreparePutRequest", reflect.TypeOf((*MockInterface)(nil).PreparePutRequest), varargs...)
 }
 
-// PreparePostRequest mocks base method.
+// PreparePostRequest mocks base method
 func (m *MockInterface) PreparePostRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{ctx}
@@ -104,14 +103,14 @@ func (m *MockInterface) PreparePostRequest(ctx context.Context, decorators ...au
 	return ret0, ret1
 }
 
-// PreparePostRequest indicates an expected call of PreparePostRequest.
+// PreparePostRequest indicates an expected call of PreparePostRequest
 func (mr *MockInterfaceMockRecorder) PreparePostRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{ctx}, decorators...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PreparePostRequest", reflect.TypeOf((*MockInterface)(nil).PreparePostRequest), varargs...)
 }
 
-// PrepareGetRequest mocks base method.
+// PrepareGetRequest mocks base method
 func (m *MockInterface) PrepareGetRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{ctx}
@@ -124,14 +123,14 @@ func (m *MockInterface) PrepareGetRequest(ctx context.Context, decorators ...aut
 	return ret0, ret1
 }
 
-// PrepareGetRequest indicates an expected call of PrepareGetRequest.
+// PrepareGetRequest indicates an expected call of PrepareGetRequest
 func (mr *MockInterfaceMockRecorder) PrepareGetRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{ctx}, decorators...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareGetRequest", reflect.TypeOf((*MockInterface)(nil).PrepareGetRequest), varargs...)
 }
 
-// PrepareDeleteRequest mocks base method.
+// PrepareDeleteRequest mocks base method
 func (m *MockInterface) PrepareDeleteRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{ctx}
@@ -144,14 +143,14 @@ func (m *MockInterface) PrepareDeleteRequest(ctx context.Context, decorators ...
 	return ret0, ret1
 }
 
-// PrepareDeleteRequest indicates an expected call of PrepareDeleteRequest.
+// PrepareDeleteRequest indicates an expected call of PrepareDeleteRequest
 func (mr *MockInterfaceMockRecorder) PrepareDeleteRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{ctx}, decorators...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareDeleteRequest", reflect.TypeOf((*MockInterface)(nil).PrepareDeleteRequest), varargs...)
 }
 
-// PrepareHeadRequest mocks base method.
+// PrepareHeadRequest mocks base method
 func (m *MockInterface) PrepareHeadRequest(ctx context.Context, decorators ...autorest.PrepareDecorator) (*http.Request, error) {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{ctx}
@@ -164,14 +163,14 @@ func (m *MockInterface) PrepareHeadRequest(ctx context.Context, decorators ...au
 	return ret0, ret1
 }
 
-// PrepareHeadRequest indicates an expected call of PrepareHeadRequest.
+// PrepareHeadRequest indicates an expected call of PrepareHeadRequest
 func (mr *MockInterfaceMockRecorder) PrepareHeadRequest(ctx interface{}, decorators ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{ctx}, decorators...)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PrepareHeadRequest", reflect.TypeOf((*MockInterface)(nil).PrepareHeadRequest), varargs...)
 }
 
-// WaitForAsyncOperationCompletion mocks base method.
+// WaitForAsyncOperationCompletion mocks base method
 func (m *MockInterface) WaitForAsyncOperationCompletion(ctx context.Context, future *azure.Future, asyncOperationName string) error {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "WaitForAsyncOperationCompletion", ctx, future, asyncOperationName)
@@ -179,13 +178,13 @@ func (m *MockInterface) WaitForAsyncOperationCompletion(ctx context.Context, fut
 	return ret0
 }
 
-// WaitForAsyncOperationCompletion indicates an expected call of WaitForAsyncOperationCompletion.
+// WaitForAsyncOperationCompletion indicates an expected call of WaitForAsyncOperationCompletion
 func (mr *MockInterfaceMockRecorder) WaitForAsyncOperationCompletion(ctx, future, asyncOperationName interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForAsyncOperationCompletion", reflect.TypeOf((*MockInterface)(nil).WaitForAsyncOperationCompletion), ctx, future, asyncOperationName)
 }
 
-// WaitForAsyncOperationResult mocks base method.
+// WaitForAsyncOperationResult mocks base method
 func (m *MockInterface) WaitForAsyncOperationResult(ctx context.Context, future *azure.Future, asyncOperationName string) (*http.Response, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "WaitForAsyncOperationResult", ctx, future, asyncOperationName)
@@ -194,13 +193,13 @@ func (m *MockInterface) WaitForAsyncOperationResult(ctx context.Context, future
 	return ret0, ret1
 }
 
-// WaitForAsyncOperationResult indicates an expected call of WaitForAsyncOperationResult.
+// WaitForAsyncOperationResult indicates an expected call of WaitForAsyncOperationResult
 func (mr *MockInterfaceMockRecorder) WaitForAsyncOperationResult(ctx, future, asyncOperationName interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForAsyncOperationResult", reflect.TypeOf((*MockInterface)(nil).WaitForAsyncOperationResult), ctx, future, asyncOperationName)
 }
 
-// SendAsync mocks base method.
+// SendAsync mocks base method
 func (m *MockInterface) SendAsync(ctx context.Context, request *http.Request) (*azure.Future, *http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "SendAsync", ctx, request)
@@ -210,13 +209,13 @@ func (m *MockInterface) SendAsync(ctx context.Context, request *http.Request) (*
 	return ret0, ret1, ret2
 }
 
-// SendAsync indicates an expected call of SendAsync.
+// SendAsync indicates an expected call of SendAsync
 func (mr *MockInterfaceMockRecorder) SendAsync(ctx, request interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendAsync", reflect.TypeOf((*MockInterface)(nil).SendAsync), ctx, request)
 }
 
-// PutResource mocks base method.
+// PutResource mocks base method
 func (m *MockInterface) PutResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "PutResource", ctx, resourceID, parameters)
@@ -225,13 +224,13 @@ func (m *MockInterface) PutResource(ctx context.Context, resourceID string, para
 	return ret0, ret1
 }
 
-// PutResource indicates an expected call of PutResource.
+// PutResource indicates an expected call of PutResource
 func (mr *MockInterfaceMockRecorder) PutResource(ctx, resourceID, parameters interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResource", reflect.TypeOf((*MockInterface)(nil).PutResource), ctx, resourceID, parameters)
 }
 
-// PutResources mocks base method.
+// PutResources mocks base method
 func (m *MockInterface) PutResources(ctx context.Context, resources map[string]interface{}) map[string]*armclient.PutResourcesResponse {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "PutResources", ctx, resources)
@@ -239,13 +238,27 @@ func (m *MockInterface) PutResources(ctx context.Context, resources map[string]i
 	return ret0
 }
 
-// PutResources indicates an expected call of PutResources.
+// PutResources indicates an expected call of PutResources
 func (mr *MockInterfaceMockRecorder) PutResources(ctx, resources interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResources", reflect.TypeOf((*MockInterface)(nil).PutResources), ctx, resources)
 }
 
-// PutResourceWithDecorators mocks base method.
+// PutResourcesInBatches mocks base method
+func (m *MockInterface) PutResourcesInBatches(ctx context.Context, resources map[string]interface{}, batchSize int) map[string]*armclient.PutResourcesResponse {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "PutResourcesInBatches", ctx, resources, batchSize)
+	ret0, _ := ret[0].(map[string]*armclient.PutResourcesResponse)
+	return ret0
+}
+
+// PutResourcesInBatches indicates an expected call of PutResourcesInBatches
+func (mr *MockInterfaceMockRecorder) PutResourcesInBatches(ctx, resources, batchSize interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResourcesInBatches", reflect.TypeOf((*MockInterface)(nil).PutResourcesInBatches), ctx, resources, batchSize)
+}
+
+// PutResourceWithDecorators mocks base method
 func (m *MockInterface) PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "PutResourceWithDecorators", ctx, resourceID, parameters, decorators)
@@ -254,13 +267,13 @@ func (m *MockInterface) PutResourceWithDecorators(ctx context.Context, resourceI
 	return ret0, ret1
 }
 
-// PutResourceWithDecorators indicates an expected call of PutResourceWithDecorators.
+// PutResourceWithDecorators indicates an expected call of PutResourceWithDecorators
 func (mr *MockInterfaceMockRecorder) PutResourceWithDecorators(ctx, resourceID, parameters, decorators interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResourceWithDecorators", reflect.TypeOf((*MockInterface)(nil).PutResourceWithDecorators), ctx, resourceID, parameters, decorators)
 }
 
-// PatchResource mocks base method.
+// PatchResource mocks base method
 func (m *MockInterface) PatchResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "PatchResource", ctx, resourceID, parameters)
@@ -269,13 +282,13 @@ func (m *MockInterface) PatchResource(ctx context.Context, resourceID string, pa
 	return ret0, ret1
 }
 
-// PatchResource indicates an expected call of PatchResource.
+// PatchResource indicates an expected call of PatchResource
 func (mr *MockInterfaceMockRecorder) PatchResource(ctx, resourceID, parameters interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PatchResource", reflect.TypeOf((*MockInterface)(nil).PatchResource), ctx, resourceID, parameters)
 }
 
-// PatchResourceAsync mocks base method.
+// PatchResourceAsync mocks base method
 func (m *MockInterface) PatchResourceAsync(ctx context.Context, resourceID string, parameters interface{}) (*azure.Future, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "PatchResourceAsync", ctx, resourceID, parameters)
@@ -284,13 +297,13 @@ func (m *MockInterface) PatchResourceAsync(ctx context.Context, resourceID strin
 	return ret0, ret1
 }
 
-// PatchResourceAsync indicates an expected call of PatchResourceAsync.
+// PatchResourceAsync indicates an expected call of PatchResourceAsync
 func (mr *MockInterfaceMockRecorder) PatchResourceAsync(ctx, resourceID, parameters interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PatchResourceAsync", reflect.TypeOf((*MockInterface)(nil).PatchResourceAsync), ctx, resourceID, parameters)
 }
 
-// PutResourceAsync mocks base method.
+// PutResourceAsync mocks base method
 func (m *MockInterface) PutResourceAsync(ctx context.Context, resourceID string, parameters interface{}) (*azure.Future, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "PutResourceAsync", ctx, resourceID, parameters)
@@ -299,13 +312,13 @@ func (m *MockInterface) PutResourceAsync(ctx context.Context, resourceID string,
 	return ret0, ret1
 }
 
-// PutResourceAsync indicates an expected call of PutResourceAsync.
+// PutResourceAsync indicates an expected call of PutResourceAsync
 func (mr *MockInterfaceMockRecorder) PutResourceAsync(ctx, resourceID, parameters interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResourceAsync", reflect.TypeOf((*MockInterface)(nil).PutResourceAsync), ctx, resourceID, parameters)
 }
 
-// HeadResource mocks base method.
+// HeadResource mocks base method
 func (m *MockInterface) HeadResource(ctx context.Context, resourceID string) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "HeadResource", ctx, resourceID)
@@ -314,13 +327,13 @@ func (m *MockInterface) HeadResource(ctx context.Context, resourceID string) (*h
 	return ret0, ret1
 }
 
-// HeadResource indicates an expected call of HeadResource.
+// HeadResource indicates an expected call of HeadResource
 func (mr *MockInterfaceMockRecorder) HeadResource(ctx, resourceID interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadResource", reflect.TypeOf((*MockInterface)(nil).HeadResource), ctx, resourceID)
 }
 
-// GetResource mocks base method.
+// GetResource mocks base method
 func (m *MockInterface) GetResource(ctx context.Context, resourceID, expand string) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetResource", ctx, resourceID, expand)
@@ -329,13 +342,13 @@ func (m *MockInterface) GetResource(ctx context.Context, resourceID, expand stri
 	return ret0, ret1
 }
 
-// GetResource indicates an expected call of GetResource.
+// GetResource indicates an expected call of GetResource
 func (mr *MockInterfaceMockRecorder) GetResource(ctx, resourceID, expand interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResource", reflect.TypeOf((*MockInterface)(nil).GetResource), ctx, resourceID, expand)
 }
 
-// GetResourceWithDecorators mocks base method.
+// GetResourceWithDecorators mocks base method
 func (m *MockInterface) GetResourceWithDecorators(ctx context.Context, resourceID string, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "GetResourceWithDecorators", ctx, resourceID, decorators)
@@ -344,13 +357,13 @@ func (m *MockInterface) GetResourceWithDecorators(ctx context.Context, resourceI
 	return ret0, ret1
 }
 
-// GetResourceWithDecorators indicates an expected call of GetResourceWithDecorators.
+// GetResourceWithDecorators indicates an expected call of GetResourceWithDecorators
 func (mr *MockInterfaceMockRecorder) GetResourceWithDecorators(ctx, resourceID, decorators interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResourceWithDecorators", reflect.TypeOf((*MockInterface)(nil).GetResourceWithDecorators), ctx, resourceID, decorators)
 }
 
-// PostResource mocks base method.
+// PostResource mocks base method
 func (m *MockInterface) PostResource(ctx context.Context, resourceID, action string, parameters interface{}, queryParameters map[string]interface{}) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "PostResource", ctx, resourceID, action, parameters, queryParameters)
@@ -359,13 +372,13 @@ func (m *MockInterface) PostResource(ctx context.Context, resourceID, action str
 	return ret0, ret1
 }
 
-// PostResource indicates an expected call of PostResource.
+// PostResource indicates an expected call of PostResource
 func (mr *MockInterfaceMockRecorder) PostResource(ctx, resourceID, action, parameters, queryParameters interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PostResource", reflect.TypeOf((*MockInterface)(nil).PostResource), ctx, resourceID, action, parameters, queryParameters)
 }
 
-// DeleteResource mocks base method.
+// DeleteResource mocks base method
 func (m *MockInterface) DeleteResource(ctx context.Context, resourceID, ifMatch string) *retry.Error {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "DeleteResource", ctx, resourceID, ifMatch)
@@ -373,13 +386,13 @@ func (m *MockInterface) DeleteResource(ctx context.Context, resourceID, ifMatch
 	return ret0
 }
 
-// DeleteResource indicates an expected call of DeleteResource.
+// DeleteResource indicates an expected call of DeleteResource
 func (mr *MockInterfaceMockRecorder) DeleteResource(ctx, resourceID, ifMatch interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteResource", reflect.TypeOf((*MockInterface)(nil).DeleteResource), ctx, resourceID, ifMatch)
 }
 
-// DeleteResourceAsync mocks base method.
+// DeleteResourceAsync mocks base method
 func (m *MockInterface) DeleteResourceAsync(ctx context.Context, resourceID, ifMatch string) (*azure.Future, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "DeleteResourceAsync", ctx, resourceID, ifMatch)
@@ -388,19 +401,19 @@ func (m *MockInterface) DeleteResourceAsync(ctx context.Context, resourceID, ifM
 	return ret0, ret1
 }
 
-// DeleteResourceAsync indicates an expected call of DeleteResourceAsync.
+// DeleteResourceAsync indicates an expected call of DeleteResourceAsync
 func (mr *MockInterfaceMockRecorder) DeleteResourceAsync(ctx, resourceID, ifMatch interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteResourceAsync", reflect.TypeOf((*MockInterface)(nil).DeleteResourceAsync), ctx, resourceID, ifMatch)
 }
 
-// CloseResponse mocks base method.
+// CloseResponse mocks base method
 func (m *MockInterface) CloseResponse(ctx context.Context, response *http.Response) {
 	m.ctrl.T.Helper()
 	m.ctrl.Call(m, "CloseResponse", ctx, response)
 }
 
-// CloseResponse indicates an expected call of CloseResponse.
+// CloseResponse indicates an expected call of CloseResponse
 func (mr *MockInterfaceMockRecorder) CloseResponse(ctx, response interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseResponse", reflect.TypeOf((*MockInterface)(nil).CloseResponse), ctx, response)
diff --git a/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go b/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go
index ea20146a2..8379f2449 100644
--- a/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go
+++ b/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go
@@ -439,7 +439,9 @@ func (page VirtualMachineScaleSetVMListResultPage) Values() []compute.VirtualMac
 }
 
 // UpdateVMs updates a list of VirtualMachineScaleSetVM from map[instanceID]compute.VirtualMachineScaleSetVM.
-func (c *Client) UpdateVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string) *retry.Error {
+// If the batch size > 0, it will send sync requests concurrently in batches, or it will send sync requests in sequence.
+// No matter what the batch size is, it will process the async requests concurrently in one single batch.
+func (c *Client) UpdateVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string, batchSize int) *retry.Error {
 	mc := metrics.NewMetricContext("vmssvm", "update_vms", resourceGroupName, c.subscriptionID, source)
 
 	// Report errors if the client is rate limited.
@@ -455,7 +457,7 @@ func (c *Client) UpdateVMs(ctx context.Context, resourceGroupName string, VMScal
 		return rerr
 	}
 
-	rerr := c.updateVMSSVMs(ctx, resourceGroupName, VMScaleSetName, instances)
+	rerr := c.updateVMSSVMs(ctx, resourceGroupName, VMScaleSetName, instances, batchSize)
 	mc.Observe(rerr)
 	if rerr != nil {
 		if rerr.IsThrottled() {
@@ -470,7 +472,7 @@ func (c *Client) UpdateVMs(ctx context.Context, resourceGroupName string, VMScal
 }
 
 // updateVMSSVMs updates a list of VirtualMachineScaleSetVM from map[instanceID]compute.VirtualMachineScaleSetVM.
-func (c *Client) updateVMSSVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM) *retry.Error {
+func (c *Client) updateVMSSVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, batchSize int) *retry.Error {
 	resources := make(map[string]interface{})
 	for instanceID, parameter := range instances {
 		resourceID := armclient.GetChildResourceID(
@@ -484,7 +486,7 @@ func (c *Client) updateVMSSVMs(ctx context.Context, resourceGroupName string, VM
 		resources[resourceID] = parameter
 	}
 
-	responses := c.armClient.PutResources(ctx, resources)
+	responses := c.armClient.PutResourcesInBatches(ctx, resources, batchSize)
 	errors := make([]*retry.Error, 0)
 	for resourceID, resp := range responses {
 		if resp == nil {
diff --git a/pkg/azureclients/vmssvmclient/interface.go b/pkg/azureclients/vmssvmclient/interface.go
index e10775cb2..84adbee11 100644
--- a/pkg/azureclients/vmssvmclient/interface.go
+++ b/pkg/azureclients/vmssvmclient/interface.go
@@ -53,5 +53,5 @@ type Interface interface {
 	WaitForUpdateResult(ctx context.Context, future *azure.Future, resourceGroupName, source string) *retry.Error
 
 	// UpdateVMs updates a list of VirtualMachineScaleSetVM from map[instanceID]compute.VirtualMachineScaleSetVM.
-	UpdateVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string) *retry.Error
+	UpdateVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string, batchSize int) *retry.Error
 }
diff --git a/pkg/azureclients/vmssvmclient/mockvmssvmclient/interface.go b/pkg/azureclients/vmssvmclient/mockvmssvmclient/interface.go
index cecf71964..f98312e9e 100644
--- a/pkg/azureclients/vmssvmclient/mockvmssvmclient/interface.go
+++ b/pkg/azureclients/vmssvmclient/mockvmssvmclient/interface.go
@@ -23,38 +23,37 @@ package mockvmssvmclient
 
 import (
 	context "context"
-	reflect "reflect"
-
 	compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-12-01/compute"
 	azure "github.com/Azure/go-autorest/autorest/azure"
 	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
 	retry "sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
-// MockInterface is a mock of Interface interface.
+// MockInterface is a mock of Interface interface
 type MockInterface struct {
 	ctrl     *gomock.Controller
 	recorder *MockInterfaceMockRecorder
 }
 
-// MockInterfaceMockRecorder is the mock recorder for MockInterface.
+// MockInterfaceMockRecorder is the mock recorder for MockInterface
 type MockInterfaceMockRecorder struct {
 	mock *MockInterface
 }
 
-// NewMockInterface creates a new mock instance.
+// NewMockInterface creates a new mock instance
 func NewMockInterface(ctrl *gomock.Controller) *MockInterface {
 	mock := &MockInterface{ctrl: ctrl}
 	mock.recorder = &MockInterfaceMockRecorder{mock}
 	return mock
 }
 
-// EXPECT returns an object that allows the caller to indicate expected use.
+// EXPECT returns an object that allows the caller to indicate expected use
 func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 	return m.recorder
 }
 
-// Get mocks base method.
+// Get mocks base method
 func (m *MockInterface) Get(ctx context.Context, resourceGroupName, VMScaleSetName, instanceID string, expand compute.InstanceViewTypes) (compute.VirtualMachineScaleSetVM, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, VMScaleSetName, instanceID, expand)
@@ -63,13 +62,13 @@ func (m *MockInterface) Get(ctx context.Context, resourceGroupName, VMScaleSetNa
 	return ret0, ret1
 }
 
-// Get indicates an expected call of Get.
+// Get indicates an expected call of Get
 func (mr *MockInterfaceMockRecorder) Get(ctx, resourceGroupName, VMScaleSetName, instanceID, expand interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockInterface)(nil).Get), ctx, resourceGroupName, VMScaleSetName, instanceID, expand)
 }
 
-// List mocks base method.
+// List mocks base method
 func (m *MockInterface) List(ctx context.Context, resourceGroupName, virtualMachineScaleSetName, expand string) ([]compute.VirtualMachineScaleSetVM, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "List", ctx, resourceGroupName, virtualMachineScaleSetName, expand)
@@ -78,13 +77,13 @@ func (m *MockInterface) List(ctx context.Context, resourceGroupName, virtualMach
 	return ret0, ret1
 }
 
-// List indicates an expected call of List.
+// List indicates an expected call of List
 func (mr *MockInterfaceMockRecorder) List(ctx, resourceGroupName, virtualMachineScaleSetName, expand interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockInterface)(nil).List), ctx, resourceGroupName, virtualMachineScaleSetName, expand)
 }
 
-// Update mocks base method.
+// Update mocks base method
 func (m *MockInterface) Update(ctx context.Context, resourceGroupName, VMScaleSetName, instanceID string, parameters compute.VirtualMachineScaleSetVM, source string) *retry.Error {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Update", ctx, resourceGroupName, VMScaleSetName, instanceID, parameters, source)
@@ -92,13 +91,13 @@ func (m *MockInterface) Update(ctx context.Context, resourceGroupName, VMScaleSe
 	return ret0
 }
 
-// Update indicates an expected call of Update.
+// Update indicates an expected call of Update
 func (mr *MockInterfaceMockRecorder) Update(ctx, resourceGroupName, VMScaleSetName, instanceID, parameters, source interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockInterface)(nil).Update), ctx, resourceGroupName, VMScaleSetName, instanceID, parameters, source)
 }
 
-// UpdateAsync mocks base method.
+// UpdateAsync mocks base method
 func (m *MockInterface) UpdateAsync(ctx context.Context, resourceGroupName, VMScaleSetName, instanceID string, parameters compute.VirtualMachineScaleSetVM, source string) (*azure.Future, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "UpdateAsync", ctx, resourceGroupName, VMScaleSetName, instanceID, parameters, source)
@@ -107,13 +106,13 @@ func (m *MockInterface) UpdateAsync(ctx context.Context, resourceGroupName, VMSc
 	return ret0, ret1
 }
 
-// UpdateAsync indicates an expected call of UpdateAsync.
+// UpdateAsync indicates an expected call of UpdateAsync
 func (mr *MockInterfaceMockRecorder) UpdateAsync(ctx, resourceGroupName, VMScaleSetName, instanceID, parameters, source interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateAsync", reflect.TypeOf((*MockInterface)(nil).UpdateAsync), ctx, resourceGroupName, VMScaleSetName, instanceID, parameters, source)
 }
 
-// WaitForUpdateResult mocks base method.
+// WaitForUpdateResult mocks base method
 func (m *MockInterface) WaitForUpdateResult(ctx context.Context, future *azure.Future, resourceGroupName, source string) *retry.Error {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "WaitForUpdateResult", ctx, future, resourceGroupName, source)
@@ -121,22 +120,22 @@ func (m *MockInterface) WaitForUpdateResult(ctx context.Context, future *azure.F
 	return ret0
 }
 
-// WaitForUpdateResult indicates an expected call of WaitForUpdateResult.
+// WaitForUpdateResult indicates an expected call of WaitForUpdateResult
 func (mr *MockInterfaceMockRecorder) WaitForUpdateResult(ctx, future, resourceGroupName, source interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForUpdateResult", reflect.TypeOf((*MockInterface)(nil).WaitForUpdateResult), ctx, future, resourceGroupName, source)
 }
 
-// UpdateVMs mocks base method.
-func (m *MockInterface) UpdateVMs(ctx context.Context, resourceGroupName, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string) *retry.Error {
+// UpdateVMs mocks base method
+func (m *MockInterface) UpdateVMs(ctx context.Context, resourceGroupName, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string, batchSize int) *retry.Error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "UpdateVMs", ctx, resourceGroupName, VMScaleSetName, instances, source)
+	ret := m.ctrl.Call(m, "UpdateVMs", ctx, resourceGroupName, VMScaleSetName, instances, source, batchSize)
 	ret0, _ := ret[0].(*retry.Error)
 	return ret0
 }
 
-// UpdateVMs indicates an expected call of UpdateVMs.
-func (mr *MockInterfaceMockRecorder) UpdateVMs(ctx, resourceGroupName, VMScaleSetName, instances, source interface{}) *gomock.Call {
+// UpdateVMs indicates an expected call of UpdateVMs
+func (mr *MockInterfaceMockRecorder) UpdateVMs(ctx, resourceGroupName, VMScaleSetName, instances, source, batchSize interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVMs", reflect.TypeOf((*MockInterface)(nil).UpdateVMs), ctx, resourceGroupName, VMScaleSetName, instances, source)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVMs", reflect.TypeOf((*MockInterface)(nil).UpdateVMs), ctx, resourceGroupName, VMScaleSetName, instances, source, batchSize)
 }
diff --git a/pkg/provider/azure.go b/pkg/provider/azure.go
index f17772237..cc841da51 100644
--- a/pkg/provider/azure.go
+++ b/pkg/provider/azure.go
@@ -240,6 +240,9 @@ type Config struct {
 	// `nodeIP`: vm private IPs will be attached to the inbound backend pool of the load balancer;
 	// `podIP`: pod IPs will be attached to the inbound backend pool of the load balancer (not supported yet).
 	LoadBalancerBackendPoolConfigurationType string `json:"loadBalancerBackendPoolConfigurationType,omitempty" yaml:"loadBalancerBackendPoolConfigurationType,omitempty"`
+	// PutVMSSVMBatchSize defines how many requests the client send concurrently when putting the VMSS VMs.
+	// If it is smaller than or equal to zero, the request will be sent one by one in sequence (default).
+	PutVMSSVMBatchSize int `json:"putVMSSVMBatchSize" yaml:"putVMSSVMBatchSize"`
 }
 
 type InitSecretConfig struct {
@@ -640,6 +643,10 @@ func (az *Cloud) isLBBackendPoolTypeNodeIP() bool {
 	return strings.EqualFold(az.LoadBalancerBackendPoolConfigurationType, consts.LoadBalancerBackendPoolConfigurationTypeNodeIP)
 }
 
+func (az *Cloud) getPutVMSSVMBatchSize() int {
+	return az.PutVMSSVMBatchSize
+}
+
 func (az *Cloud) initCaches() (err error) {
 	az.vmCache, err = az.newVMCache()
 	if err != nil {
diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go
index b2a6ad370..0a14b870c 100644
--- a/pkg/provider/azure_vmss.go
+++ b/pkg/provider/azure_vmss.go
@@ -1330,7 +1330,7 @@ func (ss *ScaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac
 			ctx, cancel := getContextWithCancel()
 			defer cancel()
 			klog.V(2).Infof("EnsureHostInPool begins to UpdateVMs for VMSS(%s, %s) with new backendPoolID %s", meta.resourceGroup, meta.vmssName, backendPoolID)
-			rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update")
+			rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update", ss.getPutVMSSVMBatchSize())
 			if rerr != nil {
 				klog.Errorf("EnsureHostInPool UpdateVMs for VMSS(%s, %s) failed with error %v", meta.resourceGroup, meta.vmssName, rerr.Error())
 				return rerr.Error()
@@ -1592,7 +1592,7 @@ func (ss *ScaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID,
 			ctx, cancel := getContextWithCancel()
 			defer cancel()
 			klog.V(2).Infof("EnsureBackendPoolDeleted begins to UpdateVMs for VMSS(%s, %s) with backendPoolID %s", meta.resourceGroup, meta.vmssName, backendPoolID)
-			rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update")
+			rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update", ss.getPutVMSSVMBatchSize())
 			if rerr != nil {
 				klog.Errorf("EnsureBackendPoolDeleted UpdateVMs for VMSS(%s, %s) failed with error %v", meta.resourceGroup, meta.vmssName, rerr.Error())
 				return rerr.Error()
diff --git a/site/content/en/blog/releases/v1.23.0.md b/site/content/en/blog/releases/v1.23.0.md
deleted file mode 100644
index ae3f3bb02..000000000
--- a/site/content/en/blog/releases/v1.23.0.md
+++ /dev/null
@@ -1,172 +0,0 @@
----
-title: "v1.23.0"
-linkTitle: "v1.23.0"
-date: 2021-12-16
-description: >
-  Cloud Provider Azure v1.23.0
----
-
-# Changelog since v1.1.0
-
-## Changes by Kind
-
-### Feature
-
-- Adds support for the Cloud Node Manager to run as a Windows Service. This can be enabled using the --windows-service flag. ([#823](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/823), [@JoelSpeed](https://github.com/JoelSpeed))
-- Chore: rename tag key for CSI drivers ([#799](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/799), [@andyzhangx](https://github.com/andyzhangx))
-- Feat: Adds windows server 2022 build ([#821](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/821), [@nick5616](https://github.com/nick5616))
-- Feat: add AccessTier in file share creation interface ([#781](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/781), [@andyzhangx](https://github.com/andyzhangx))
-- Feat: add AllowBlobPublicAccess setting in storage account creation ([#784](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/784), [@andyzhangx](https://github.com/andyzhangx))
-- Feat: add disk online resize support ([#889](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/889), [@andyzhangx](https://github.com/andyzhangx))
-- Feat: only restart all controllers after stopping if needed ([#832](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/832), [@nilo19](https://github.com/nilo19))
-- Feat: reduce vmss cache refresh in parallel disk attach/detach ([#803](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/803), [@andyzhangx](https://github.com/andyzhangx))
-- Feat: support creating account if not exists when account name is provided ([#786](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/786), [@andyzhangx](https://github.com/andyzhangx))
-- Feat: support json style tags ([#892](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/892), [@nilo19](https://github.com/nilo19))
-- Feat: support reloading the cloud controller manager based on the changes of the config file ([#769](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/769), [@nilo19](https://github.com/nilo19))
-- Introduce a new config `loadBalancerBackendPoolConfigurationType` and it can be set to `nodeIPConfiguration` (default) or `nodeIP`. If set to `nodeIPConfiguration`, everything will keep unchanged. If set to `nodeIP`, the cloud provider will call the LB API to attach the node private IPs to the LB instead of linking the NICs to the LB. ([#918](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/918), [@nilo19](https://github.com/nilo19))
-- Prefix 'k8s-azure-' has been added to the following tags:
-  
-  Legacy Tag | New Tag | Comment
-  -- | -- | --
-  service | k8s-azure-service | Applied on public IP
-  kubernetes-cluster-name | k8s-azure-cluster-name | Applied on public IP
-  kubernetes-dns-label-service | k8s-azure-dns-label-service | Applied on public IP
-  
-  To keep backward compability, the legacy tags on existing public IP would not be removed, but newly created public IPs would only get the new tags. ([#815](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/815), [@feiskyer](https://github.com/feiskyer))
-
-### Documentation
-
-- Chore: add release notes for v0.7.9, v1.0.6 and v1.1.2 ([#867](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/867), [@nilo19](https://github.com/nilo19))
-
-### Failing Test
-
-- Chore: fix implictly required vendor ([#775](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/775), [@nilo19](https://github.com/nilo19))
-
-### Bug or Regression
-
-- **Release note**: ([#780](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/780), [@andyzhangx](https://github.com/andyzhangx))
-- 1 controllerserver.go:378] delete azure disk(/subscriptions/xxx/resourceGroups/aks55h93-nodegroup/providers/Microsoft.Compute/disks/pvc-ecfefbb9-102c-4f40-944f-d5d4b6183568) returned with Retriable: true, RetryAfter: 0s, HTTPStatusCode: 0, RawError: azure cloud provider rate limited(read) for operation "GetDisk"
-   1 utils.go:100] GRPC error: Retriable: true, RetryAfter: 0s, HTTPStatusCode: 0, RawError: azure cloud provider rate limited(read) for operation "GetDisk" ([#757](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/757), [@andyzhangx](https://github.com/andyzhangx))
-- Fix detach disk issue on deleting vmss node ([#774](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/774), [@andyzhangx](https://github.com/andyzhangx))
-- Fix: consolidate logs for instance not found error ([#794](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/794), [@feiskyer](https://github.com/feiskyer))
-- Fix: detach disk should return error when throttled ([#929](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/929), [@andyzhangx](https://github.com/andyzhangx))
-- Fix: do not crash if the region does not support zones ([#850](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/850), [@nilo19](https://github.com/nilo19))
-- Fix: do not delete the lb that does not exist ([#860](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/860), [@nilo19](https://github.com/nilo19))
-- Fix: fix a potential data race issue in unit test ([#848](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/848), [@nilo19](https://github.com/nilo19))
-- Fix: ignore the case when comparing azure tags in service annotation ([#791](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/791), [@nilo19](https://github.com/nilo19))
-- Fix: multi accounts creation issue when private endpoint creation failed ([#880](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/880), [@andyzhangx](https://github.com/andyzhangx))
-- Fix: panic due to nil pointer ([#899](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/899), [@andyzhangx](https://github.com/andyzhangx))
-- Fix: remove VMSS instances from SLB backend pool only when they are explicitly labeled
-  fix: remove VMSS from SLB backend pool only when necessary ([#856](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/856), [@feiskyer](https://github.com/feiskyer))
-- Fix: remove outdated ipv4 route when the corresponding node is deleted ([#876](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/876), [@nilo19](https://github.com/nilo19))
-- Fix: use correct gitVersion in userAgent ([#764](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/764), [@feiskyer](https://github.com/feiskyer))
-- Version.Info{Major:"1", Minor:"0+", GitVersion:"v1.0.1-116-g93dee8c35", GitCommit:"93dee8c35bddbff6beabb764725c74d252130c3c", GitTreeState:"", BuildDate:"2021-08-11T08:21:10Z", GoVersion:"go1.16.6", Compiler:"gc", Platform:"darwin/amd64"} ([#749](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/749), [@feiskyer](https://github.com/feiskyer))
-
-### Other (Cleanup or Flake)
-
-- Chore: add mixed protocol service e2e test ([#897](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/897), [@nilo19](https://github.com/nilo19))
-- Chore: bump k8s.io/cloud-provider to v0.23.0 ([#934](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/934), [@nilo19](https://github.com/nilo19))
-- Chore: detect data race in unit tests ([#849](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/849), [@nilo19](https://github.com/nilo19))
-- Chore: expose some public functions ([#754](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/754), [@andyzhangx](https://github.com/andyzhangx))
-- Chore: logging the service body when service controller invokes the cloud provider ([#778](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/778), [@nilo19](https://github.com/nilo19))
-- Cleanup: remove blob disk controller
-  blob disk  is not supported any more ([#909](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/909), [@andyzhangx](https://github.com/andyzhangx))
-- Fix: report an error when route table name is not configured ([#819](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/819), [@feiskyer](https://github.com/feiskyer))
-
-### Uncategorized
-
-- Fix: skip instance not found when decoupling vmss from lb ([#842](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/842), [@nilo19](https://github.com/nilo19))
-- Fix: switch to sync detach disk ([#790](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/790), [@andyzhangx](https://github.com/andyzhangx))
-- Use docker buildx for multi-arch node image ([#855](https://github.com/kubernetes-sigs/cloud-provider-azure/pull/855), [@mainred](https://github.com/mainred))
-
-## Dependencies
-
-### Added
-- github.com/cncf/xds/go: [fbca930](https://github.com/cncf/xds/go/tree/fbca930)
-- github.com/getkin/kin-openapi: [v0.76.0](https://github.com/getkin/kin-openapi/tree/v0.76.0)
-- github.com/go-logr/zapr: [v1.2.0](https://github.com/go-logr/zapr/tree/v1.2.0)
-- github.com/golang-jwt/jwt/v4: [v4.0.0](https://github.com/golang-jwt/jwt/v4/tree/v4.0.0)
-- github.com/gorilla/mux: [v1.8.0](https://github.com/gorilla/mux/tree/v1.8.0)
-- k8s.io/component-helpers: v0.23.0
-- k8s.io/kubelet: v0.23.0
-- sigs.k8s.io/json: c049b76
-
-### Changed
-- github.com/Azure/azure-sdk-for-go: [v55.0.0+incompatible → v55.8.0+incompatible](https://github.com/Azure/azure-sdk-for-go/compare/v55.0.0...v55.8.0)
-- github.com/Azure/go-autorest/autorest/adal: [v0.9.14 → v0.9.17](https://github.com/Azure/go-autorest/autorest/adal/compare/v0.9.14...v0.9.17)
-- github.com/Azure/go-autorest/autorest/validation: [v0.1.0 → v0.3.1](https://github.com/Azure/go-autorest/autorest/validation/compare/v0.1.0...v0.3.1)
-- github.com/Azure/go-autorest/autorest: [v0.11.19 → v0.11.22](https://github.com/Azure/go-autorest/autorest/compare/v0.11.19...v0.11.22)
-- github.com/benbjohnson/clock: [v1.0.3 → v1.1.0](https://github.com/benbjohnson/clock/compare/v1.0.3...v1.1.0)
-- github.com/envoyproxy/go-control-plane: [668b12f → 63b5d3c](https://github.com/envoyproxy/go-control-plane/compare/668b12f...63b5d3c)
-- github.com/evanphx/json-patch: [v4.11.0+incompatible → v4.12.0+incompatible](https://github.com/evanphx/json-patch/compare/v4.11.0...v4.12.0)
-- github.com/fsnotify/fsnotify: [v1.4.9 → v1.5.1](https://github.com/fsnotify/fsnotify/compare/v1.4.9...v1.5.1)
-- github.com/go-logr/logr: [v0.4.0 → v1.2.0](https://github.com/go-logr/logr/compare/v0.4.0...v1.2.0)
-- github.com/json-iterator/go: [v1.1.11 → v1.1.12](https://github.com/json-iterator/go/compare/v1.1.11...v1.1.12)
-- github.com/mitchellh/go-homedir: [v1.1.0 → v1.0.0](https://github.com/mitchellh/go-homedir/compare/v1.1.0...v1.0.0)
-- github.com/modern-go/reflect2: [v1.0.1 → v1.0.2](https://github.com/modern-go/reflect2/compare/v1.0.1...v1.0.2)
-- github.com/onsi/ginkgo: [v1.16.4 → v1.16.5](https://github.com/onsi/ginkgo/compare/v1.16.4...v1.16.5)
-- github.com/onsi/gomega: [v1.15.0 → v1.16.0](https://github.com/onsi/gomega/compare/v1.15.0...v1.16.0)
-- github.com/prometheus/common: [v0.26.0 → v0.28.0](https://github.com/prometheus/common/compare/v0.26.0...v0.28.0)
-- github.com/stretchr/objx: [v0.2.0 → v0.1.1](https://github.com/stretchr/objx/compare/v0.2.0...v0.1.1)
-- github.com/yuin/goldmark: [v1.3.5 → v1.4.0](https://github.com/yuin/goldmark/compare/v1.3.5...v1.4.0)
-- go.uber.org/zap: v1.17.0 → v1.19.0
-- golang.org/x/crypto: 5ea612d → 089bfa5
-- golang.org/x/net: 37e1c6a → e898025
-- golang.org/x/oauth2: 2e8d934 → 2bc19b1
-- golang.org/x/sys: 59db8d7 → f4d4317
-- golang.org/x/term: 6a3ed07 → 6886f2d
-- golang.org/x/text: v0.3.6 → v0.3.7
-- golang.org/x/tools: v0.1.2 → d4cc65f
-- google.golang.org/genproto: f16073e → fe13028
-- google.golang.org/grpc: v1.38.0 → v1.40.0
-- google.golang.org/protobuf: v1.26.0 → v1.27.1
-- k8s.io/api: v0.22.0 → v0.23.0
-- k8s.io/apimachinery: v0.22.0 → v0.23.0
-- k8s.io/apiserver: v0.22.0 → v0.23.0
-- k8s.io/client-go: v0.22.0 → v0.23.0
-- k8s.io/cloud-provider: v0.22.0 → v0.23.0
-- k8s.io/component-base: v0.22.0 → v0.23.0
-- k8s.io/controller-manager: v0.22.0 → v0.23.0
-- k8s.io/gengo: 3a45101 → 485abfe
-- k8s.io/klog/v2: v2.10.0 → v2.30.0
-- k8s.io/kube-openapi: 9528897 → e816edb
-- k8s.io/utils: 4b05e18 → cb0fa31
-- sigs.k8s.io/apiserver-network-proxy/konnectivity-client: v0.0.22 → v0.0.25
-- sigs.k8s.io/yaml: v1.2.0 → v1.3.0
-
-### Removed
-- cloud.google.com/go/datastore: v1.1.0
-- cloud.google.com/go/pubsub: v1.3.1
-- github.com/alecthomas/template: [fb15b89](https://github.com/alecthomas/template/tree/fb15b89)
-- github.com/alecthomas/units: [f65c72e](https://github.com/alecthomas/units/tree/f65c72e)
-- github.com/client9/misspell: [v0.3.4](https://github.com/client9/misspell/tree/v0.3.4)
-- github.com/coreos/bbolt: [v1.3.2](https://github.com/coreos/bbolt/tree/v1.3.2)
-- github.com/coreos/etcd: [v3.3.13+incompatible](https://github.com/coreos/etcd/tree/v3.3.13)
-- github.com/coreos/go-systemd: [95778df](https://github.com/coreos/go-systemd/tree/95778df)
-- github.com/coreos/pkg: [399ea9e](https://github.com/coreos/pkg/tree/399ea9e)
-- github.com/dgrijalva/jwt-go: [v3.2.0+incompatible](https://github.com/dgrijalva/jwt-go/tree/v3.2.0)
-- github.com/dgryski/go-sip13: [e10d5fe](https://github.com/dgryski/go-sip13/tree/e10d5fe)
-- github.com/dnaeon/go-vcr: [v1.1.0](https://github.com/dnaeon/go-vcr/tree/v1.1.0)
-- github.com/go-gl/glfw: [e6da0ac](https://github.com/go-gl/glfw/tree/e6da0ac)
-- github.com/gofrs/uuid: [v4.0.0+incompatible](https://github.com/gofrs/uuid/tree/v4.0.0)
-- github.com/google/martian: [v2.1.0+incompatible](https://github.com/google/martian/tree/v2.1.0)
-- github.com/hpcloud/tail: [v1.0.0](https://github.com/hpcloud/tail/tree/v1.0.0)
-- github.com/jpillora/backoff: [v1.0.0](https://github.com/jpillora/backoff/tree/v1.0.0)
-- github.com/konsorten/go-windows-terminal-sequences: [v1.0.3](https://github.com/konsorten/go-windows-terminal-sequences/tree/v1.0.3)
-- github.com/kr/logfmt: [b84e30a](https://github.com/kr/logfmt/tree/b84e30a)
-- github.com/kr/pty: [v1.1.1](https://github.com/kr/pty/tree/v1.1.1)
-- github.com/modocache/gover: [b58185e](https://github.com/modocache/gover/tree/b58185e)
-- github.com/oklog/ulid: [v1.3.1](https://github.com/oklog/ulid/tree/v1.3.1)
-- github.com/prometheus/tsdb: [v0.7.1](https://github.com/prometheus/tsdb/tree/v0.7.1)
-- github.com/rubiojr/go-vhd: [02e2102](https://github.com/rubiojr/go-vhd/tree/02e2102)
-- gopkg.in/fsnotify.v1: v1.4.7
-- gopkg.in/resty.v1: v1.12.0
-- rsc.io/binaryregexp: v0.2.0
-- rsc.io/quote/v3: v3.1.0
-- rsc.io/sampler: v1.3.0
-ree/02e2102)
-- gopkg.in/fsnotify.v1: v1.4.7
-- gopkg.in/resty.v1: v1.12.0
-- rsc.io/binaryregexp: v0.2.0
-- rsc.io/quote/v3: v3.1.0
-- rsc.io/sampler: v1.3.0
diff --git a/site/content/en/install/configs.md b/site/content/en/install/configs.md
index 1ffc9c8fe..e1ebfdec9 100644
--- a/site/content/en/install/configs.md
+++ b/site/content/en/install/configs.md
@@ -65,52 +65,50 @@ If more than one value is set, the order is `Managed Identity` > `Service Princi
 
 ## Cluster config
 
-| Name                                                       | Description                                                                                                                                                                                                       | Remark                                                                                                                                |
-|------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
-| resourceGroup                                              | The name of the resource group that the cluster is deployed in                                                                                                                                                    ||
-| location                                                   | The location of the resource group that the cluster is deployed in                                                                                                                                                ||
-| vnetName                                                   | The name of the VNet that the cluster is deployed in                                                                                                                                                              ||
-| vnetResourceGroup                                          | The name of the resource group that the Vnet is deployed in                                                                                                                                                       ||
-| subnetName                                                 | The name of the subnet that the cluster is deployed in                                                                                                                                                            ||
-| securityGroupName                                          | The name of the security group attached to the cluster's subnet                                                                                                                                                   ||
-| securityGroupResourceGroup                                 | The name of the resource group that the security group is deployed in                                                                                                                                             ||
-| routeTableName                                             | The name of the route table attached to the subnet that the cluster is deployed in                                                                                                                                | Optional in 1.6                                                                                                                       |
-| primaryAvailabilitySetName[*](#primaryavailabilitysetname) | The name of the availability set that should be used as the load balancer backend                                                                                                                                 | Optional                                                                                                                              |
-| vmType                                                     | The type of azure nodes. Candidate values are: `vmss` and `standard`                                                                                                                                              | Optional, default to `standard`                                                                                                       |
-| primaryScaleSetName[*](#primaryscalesetname)               | The name of the scale set that should be used as the load balancer backend                                                                                                                                        | Optional                                                                                                                              |
-| cloudProviderBackoff                                       | Enable exponential backoff to manage resource request retries                                                                                                                                                     | Boolean value, default to false                                                                                                       |
-| cloudProviderBackoffRetries                                | Backoff retry limit                                                                                                                                                                                               | Integer value, valid if `cloudProviderBackoff` is true                                                                                |
-| cloudProviderBackoffExponent                               | Backoff exponent                                                                                                                                                                                                  | Float value, valid if `cloudProviderBackoff` is true                                                                                  |
-| cloudProviderBackoffDuration                               | Backoff duration                                                                                                                                                                                                  | Integer value, valid if `cloudProviderBackoff` is true                                                                                |
-| cloudProviderBackoffJitter                                 | Backoff jitter                                                                                                                                                                                                    | Float value, valid if `cloudProviderBackoff` is true                                                                                  |
-| cloudProviderBackoffMode                                   | Backoff mode, supported values are "v2" and "default". Note that "v2" has been deprecated since v1.18.0.                                                                                                          | Default to "default"                                                                                                                  |
-| cloudProviderRateLimit                                     | Enable rate limiting                                                                                                                                                                                              | Boolean value, default to false                                                                                                       |
-| cloudProviderRateLimitQPS                                  | Rate limit QPS (Read)                                                                                                                                                                                             | Float value, valid if `cloudProviderRateLimit` is true                                                                                |
-| cloudProviderRateLimitBucket                               | Rate limit Bucket Size                                                                                                                                                                                            | Integar value, valid if `cloudProviderRateLimit` is true                                                                              |
-| cloudProviderRateLimitQPSWrite                             | Rate limit QPS (Write)                                                                                                                                                                                            | Float value, valid if `cloudProviderRateLimit` is true                                                                                |
-| cloudProviderRateLimitBucketWrite                          | Rate limit Bucket Size                                                                                                                                                                                            | Integer value, valid if `cloudProviderRateLimit` is true                                                                              |
-| useInstanceMetadata                                        | Use instance metadata service where possible                                                                                                                                                                      | Boolean value, default to false                                                                                                       |
-| loadBalancerSku                                            | Sku of Load Balancer and Public IP. Candidate values are: `basic` and `standard`.                                                                                                                                 | Default to `basic`.                                                                                                                   |
-| excludeMasterFromStandardLB                                | ExcludeMasterFromStandardLB excludes master nodes from standard load balancer.                                                                                                                                    | Boolean value, default to true.                                                                                                       |
-| disableOutboundSNAT                                        | Disable outbound SNAT for SLB                                                                                                                                                                                     | Default to false and available since v1.11.9, v1.12.7, v1.13.5 and v1.14.0                                                            |
-| maximumLoadBalancerRuleCount                               | Maximum allowed LoadBalancer Rule Count is the limit enforced by Azure Load balancer                                                                                                                              | Integer value, default to [148](https://github.com/kubernetes/kubernetes/blob/v1.10.0/pkg/cloudprovider/providers/azure/azure.go#L48) |
-| routeTableResourceGroup                                    | The resource group name for routeTable                                                                                                                                                                            | Default same as resourceGroup and available since v1.15.0                                                                             |
-| loadBalancerName                                           | Working together with loadBalancerResourceGroup to determine the LB name in a different resource group                                                                                                            | Since v1.18.0, default is cluster name setting on kube-controller-manager                                                             |
-| loadBalancerResourceGroup                                  | The load balancer resource group name, which is different from node resource group                                                                                                                                | Since v1.18.0, default is same as resourceGroup                                                                                       |
-| disableAvailabilitySetNodes                                | Disable supporting for AvailabilitySet virtual machines in vmss cluster. It should be only used when vmType is "vmss" and all the nodes (including master) are VMSS virtual machines                              | Since v1.18.0, default is false                                                                                                       |
-| availabilitySetNodesCacheTTLInSeconds                      | Cache TTL in seconds for availabilitySet Nodes                                                                                                                                                                    | Since v1.18.0, default is 900                                                                                                         |
-| vmssCacheTTLInSeconds                                      | Cache TTL in seconds for VMSS                                                                                                                                                                                     | Since v1.18.0, default is 600                                                                                                         |
-| vmssVirtualMachinesCacheTTLInSeconds                       | Cache TTL in seconds for VMSS virtual machines                                                                                                                                                                    | Since v1.18.0, default is 600                                                                                                         |
-| vmCacheTTLInSeconds                                        | Cache TTL in seconds for virtual machines                                                                                                                                                                         | Since v1.18.0, default is 60                                                                                                          |
-| loadBalancerCacheTTLInSeconds                              | Cache TTL in seconds for load balancers                                                                                                                                                                           | Since v1.18.0, default is 120                                                                                                         |
-| nsgCacheTTLInSeconds                                       | Cache TTL in seconds for network security group                                                                                                                                                                   | Since v1.18.0, default is 120                                                                                                         |
-| routeTableCacheTTLInSeconds                                | Cache TTL in seconds for route table                                                                                                                                                                              | Since v1.18.0, default is 120                                                                                                         |
-| disableAzureStackCloud                                     | DisableAzureStackCloud disables AzureStackCloud support. It should be used when setting Cloud with "AZURESTACKCLOUD" to customize ARM endpoints while the cluster is not running on AzureStack. Default is false. | Optional. Supported since v1.20.0 in out-of-tree cloud provider Azure.                                                                |
-| tags                                                       | Tags that would be tagged onto the cloud provider managed resources, including lb, public IP, network security group and route table.                                                                             | Optional. Supported since v1.20.0.                                                                                                    |
-| tagsMap                                                    | JSON-style tags, will be merged with `tags`                                                                                                                                                                       | Optional. Supported since v1.23.0.                                                                                                    |
-| systemTags                                                 | Tag keys that should not be deleted when being updated.                                                                                                                                                           | Optional. Supported since v1.21.0.                                                                                                    |
-| enableMultipleStandardLoadBalancers                        | Enable multiple standard Load Balancers per cluster.                                                                                                                                                              | Optional. Supported since v1.20.0                                                                                                     |
-| loadBalancerBackendPoolConfigurationType                   | The type of the Load Balancer backend pool. Supported values are `nodeIPConfiguration` (default) and `nodeIP`                                                                                                     | Optional. Supported since v1.23.0                                                                                                     |
+|Name|Description|Remark|
+|---|---|---|
+|resourceGroup|The name of the resource group that the cluster is deployed in||
+|location|The location of the resource group that the cluster is deployed in||
+|vnetName|The name of the VNet that the cluster is deployed in||
+|vnetResourceGroup|The name of the resource group that the Vnet is deployed in||
+|subnetName|The name of the subnet that the cluster is deployed in||
+|securityGroupName|The name of the security group attached to the cluster's subnet||
+|securityGroupResourceGroup|The name of the resource group that the security group is deployed in||
+|routeTableName|The name of the route table attached to the subnet that the cluster is deployed in|Optional in 1.6|
+|primaryAvailabilitySetName[*](#primaryavailabilitysetname)|The name of the availability set that should be used as the load balancer backend|Optional|
+|vmType|The type of azure nodes. Candidate values are: `vmss` and `standard`|Optional, default to `standard`|
+|primaryScaleSetName[*](#primaryscalesetname)|The name of the scale set that should be used as the load balancer backend|Optional|
+|cloudProviderBackoff|Enable exponential backoff to manage resource request retries|Boolean value, default to false|
+|cloudProviderBackoffRetries|Backoff retry limit|Integer value, valid if `cloudProviderBackoff` is true|
+|cloudProviderBackoffExponent|Backoff exponent|Float value, valid if `cloudProviderBackoff` is true|
+|cloudProviderBackoffDuration|Backoff duration|Integer value, valid if `cloudProviderBackoff` is true|
+|cloudProviderBackoffJitter|Backoff jitter|Float value, valid if `cloudProviderBackoff` is true|
+|cloudProviderBackoffMode|Backoff mode, supported values are "v2" and "default". Note that "v2" has been deprecated since v1.18.0. |Default to "default"|
+|cloudProviderRateLimit|Enable rate limiting|Boolean value, default to false|
+|cloudProviderRateLimitQPS|Rate limit QPS (Read)|Float value, valid if `cloudProviderRateLimit` is true|
+|cloudProviderRateLimitBucket|Rate limit Bucket Size|Integar value, valid if `cloudProviderRateLimit` is true|
+|cloudProviderRateLimitQPSWrite|Rate limit QPS (Write)|Float value, valid if `cloudProviderRateLimit` is true|
+|cloudProviderRateLimitBucketWrite|Rate limit Bucket Size|Integer value, valid if `cloudProviderRateLimit` is true|
+|useInstanceMetadata|Use instance metadata service where possible|Boolean value, default to false|
+|loadBalancerSku|Sku of Load Balancer and Public IP. Candidate values are: `basic` and `standard`.|Default to `basic`.|
+|excludeMasterFromStandardLB|ExcludeMasterFromStandardLB excludes master nodes from standard load balancer.|Boolean value, default to true.|
+|disableOutboundSNAT| Disable outbound SNAT for SLB | Default to false and available since v1.11.9, v1.12.7, v1.13.5 and v1.14.0|
+|maximumLoadBalancerRuleCount|Maximum allowed LoadBalancer Rule Count is the limit enforced by Azure Load balancer|Integer value, default to [148](https://github.com/kubernetes/kubernetes/blob/v1.10.0/pkg/cloudprovider/providers/azure/azure.go#L48)|
+|routeTableResourceGroup| The resource group name for routeTable | Default same as resourceGroup and available since v1.15.0 |
+|loadBalancerName| Working together with loadBalancerResourceGroup to determine the LB name in a different resource group | Since v1.18.0, default is cluster name setting on kube-controller-manager|
+|loadBalancerResourceGroup | The load balancer resource group name, which is different from node resource group | Since v1.18.0, default is same as resourceGroup|
+|disableAvailabilitySetNodes| Disable supporting for AvailabilitySet virtual machines in vmss cluster. It should be only used when vmType is "vmss" and all the nodes (including master) are VMSS virtual machines | Since v1.18.0, default is false|
+|availabilitySetNodesCacheTTLInSeconds|Cache TTL in seconds for availabilitySet Nodes|Since v1.18.0, default is 900|
+|vmssCacheTTLInSeconds|Cache TTL in seconds for VMSS|Since v1.18.0, default is 600|
+|vmssVirtualMachinesCacheTTLInSeconds|Cache TTL in seconds for VMSS virtual machines|Since v1.18.0, default is 600|
+|vmCacheTTLInSeconds|Cache TTL in seconds for virtual machines|Since v1.18.0, default is 60|
+|loadBalancerCacheTTLInSeconds|Cache TTL in seconds for load balancers|Since v1.18.0, default is 120|
+|nsgCacheTTLInSeconds|Cache TTL in seconds for network security group|Since v1.18.0, default is 120|
+|routeTableCacheTTLInSeconds|Cache TTL in seconds for route table|Since v1.18.0, default is 120|
+| disableAzureStackCloud | DisableAzureStackCloud disables AzureStackCloud support. It should be used when setting Cloud with "AZURESTACKCLOUD" to customize ARM endpoints while the cluster is not running on AzureStack. Default is false. |Optional. Supported since v1.20.0 in out-of-tree cloud provider Azure. |
+| tags | tags that would be tagged onto the cloud provider managed resources, including lb, public IP, network security group and route table. | Optional. Supported since v1.20.0. |
+| tagsMap | JSON-style tags, will be merged with `tags` | Optional. Supported since v1.23.0. |
+| systemTags | tag keys that should not be deleted when being updated. | Optional. Supported since v1.21.0. |
 
 ### primaryAvailabilitySetName
 
diff --git a/site/content/en/topics/custom-health-probe.md b/site/content/en/topics/custom-health-probe.md
new file mode 100644
index 000000000..ea72e8155
--- /dev/null
+++ b/site/content/en/topics/custom-health-probe.md
@@ -0,0 +1,26 @@
+---
+title: "Custom load balancer health probe"
+linkTitle: "Custom load balancer health probe"
+weight: 9
+type: docs
+---
+
+Currently, the protocol of the health probe is defined as below:
+
+1. for local services, HTTP and /healthz would be used;
+1. for cluster TCP services, TCP would be used;
+1. for cluster UDP services, no health probes.
+
+Since v1.20, two service annotations `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` and `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` are introduced, which determine the new health probe behavior. If the `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` is set, both local and cluster TCP services would use the specified health probe protocol. If the `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` is set, the specified request path would be used instead of `/healthz`. Note that the request path would be ignored when using TCP or the `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` is empty. More specifically:
+
+| `externalTrafficPolicy` | `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` | `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` | protocol | request path |
+| ------------------------------------------------------------ | ---------------------------- | ------------------------------------------------------------ |------| ----- |
+| local |  | (ignored) | http | `/healthz` |
+| local | tcp | (ignored) | tcp | null |
+| local | http/https | | http/https | `/healthz` |
+| local | http/https | `/custom-path` | http/https | `/custom-path` |
+| cluster |  | (ignored) | tcp | null |
+| cluster | tcp | (ignored) | tcp | null |
+| cluster | http/https | | http/https | `/healthz` |
+| cluster | http/https | `/custom-path` | http/https | `/custom-path` |
+
diff --git a/site/content/en/topics/loadbalancer.md b/site/content/en/topics/loadbalancer.md
index dc1c5d8ab..d8b588a91 100644
--- a/site/content/en/topics/loadbalancer.md
+++ b/site/content/en/topics/loadbalancer.md
@@ -125,59 +125,6 @@ spec:
       targetPort: 30102
 ```
 
-## Multiple Standard LoadBalancer per cluster
-
-> This feature is supported since v1.20.0.
-
-There is only one external and one internal Standard Load Balancer (SLB) at most per cluster. Set `enableMultipleStandardLoadBalancers=true` in the cloud config if you want to turn on the multiple SLB mode. Similar to the basic LB, there will be a 1:1 mapping between each SLB and VMSS/VMAS. The SLB of the primary VMSS/VMAS will be named after `clusterName` in the cloud config (in AKS, it would be `kubernetes`) while the name of those belonging to non-primary VMSS/VMAS will be the name of the corresponding vmSet.
-
-> If the cluster provisioning tools like ASK-Engine and CAPZ don't proactively create a dedicated SLB for each VMSS/VMAS when enabling multiple SLB, only the primary SLB would be created. You could manually trigger the creation by setting the service annotation `service.beta.kubernetes.io/azure-load-balancer-mode` to bind the service to that VMSS/VMAS. The dedicated SLB would be created once the service reconcile loop is done. Unlike the primary SLB, there is no default outbound rules/IPs for the non-primary SLBs. That means the SLB would be deleted once all the services referencing it are deleted.
-
-### Choose which SLB to use
-
-The service annotation `service.beta.kubernetes.io/azure-load-balancer-mode` will be respected as long as `enableMultipleStandardLoadBalancers=true` when using standard LB, and the usage is the same as it is in the basic LB clusters. Specifically, there are three selection mode: `default` to select the primary SLB; `__auto__` to select the SLB with minimum rules and `vmSetName` to select the dedicated SLB of that VMSS/VMAS.
-
-### Outbound Connections of non-primary VMSS/VMAS
-
-The outbound rules of the non-primary SLB are not managed by cloud provider azure. Instead, it should be managed by cluster provisioning tools. For now, there is no outbound configuration for the non-primary VMSS/VMAS, but we plan to support customized outbound configurations in AKS and CAPZ in the future.
-
-### Sharing the primary SLB with multiple VMSS/VMAS
-
-> This feature is supported since v1.21.0
-
-For each non-primary VMSS/VMAS, one can determine to use dedicated SLB or share the primary SLB. If the VMSS/VMAS names are in the cloud config `nodepoolsWithoutDedicatedSLB`, those would join the backend pool of the primary SLB while the others would remain to have dedicated SLBs. If the VMSS/VMAS supposed to share the primary SLB owns a dedicated SLB, the dedicated one would be deleted, and the VMSS/VMAS would be joint the primary SLB's backend pool.
-
-## Custom Load Balancer health probe
-
-Currently, the protocol of the health probe is defined as below:
-
-1. for local services, HTTP and /healthz would be used;
-1. for cluster TCP services, TCP would be used;
-1. for cluster UDP services, no health probes.
-
-Since v1.20, two service annotations `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` and `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` are introduced, which determine the new health probe behavior. If the `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` is set, both local and cluster TCP services would use the specified health probe protocol. If the `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` is set, the specified request path would be used instead of `/healthz`. Note that the request path would be ignored when using TCP or the `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` is empty. More specifically:
-
-| `externalTrafficPolicy` | `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` | `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` | protocol | request path |
-| ------------------------------------------------------------ | ---------------------------- | ------------------------------------------------------------ |------| ----- |
-| local |  | (ignored) | http | `/healthz` |
-| local | tcp | (ignored) | tcp | null |
-| local | http/https | | http/https | `/healthz` |
-| local | http/https | `/custom-path` | http/https | `/custom-path` |
-| cluster |  | (ignored) | tcp | null |
-| cluster | tcp | (ignored) | tcp | null |
-| cluster | http/https | | http/https | `/healthz` |
-| cluster | http/https | `/custom-path` | http/https | `/custom-path` |
-
-## Configure Load Balancer backend
-
-> This feature is supported since v1.23.0
-
-The backend pool type can be configured by specifying `loadBalancerBackendPoolConfigurationType` in the cloud configuration file. There are three possible values:
-
-1. `nodeIPConfiguration` (default). In this case we attach nodes to the LB by calling the VMSS/NIC API to associate the corresponding node IP configuration with the LB backend pool.
-2. `nodeIP`. In this case we attach nodes to the LB by calling the LB API to add the node private IP addresses to the LB backend pool.
-3. `podIP` (not supported yet). In this case we do not attach nodes to the LB. Instead we directly adding pod IPs to the LB backend pool.
-
 ## Load balancer limits
 
 The limits of the load balancer related resources are listed below:
diff --git a/site/content/en/topics/multiple-slb.md b/site/content/en/topics/multiple-slb.md
new file mode 100644
index 000000000..638b846dc
--- /dev/null
+++ b/site/content/en/topics/multiple-slb.md
@@ -0,0 +1,26 @@
+---
+title: "Multiple Standard LoadBalancer per cluster"
+linkTitle: "Multiple Standard LoadBalancer per cluster"
+weight: 8
+type: docs
+---
+
+> This feature is supported since v1.20.0.
+
+There is only one external and one internal Standard Load Balancer (SLB) at most per cluster. Set `enableMultipleStandardLoadBalancers=true` in the cloud config if you want to turn on the multiple SLB mode. Similar to the basic LB, there will be a 1:1 mapping between each SLB and VMSS/VMAS. The SLB of the primary VMSS/VMAS will be named after `clusterName` in the cloud config (in AKS, it would be `kubernetes`) while the name of those belonging to non-primary VMSS/VMAS will be the name of the corresponding vmSet.
+
+> If the cluster provisioning tools like ASK-Engine and CAPZ don't proactively create a dedicated SLB for each VMSS/VMAS when enabling multiple SLB, only the primary SLB would be created. You could manually trigger the creation by setting the service annotation `service.beta.kubernetes.io/azure-load-balancer-mode` to bind the service to that VMSS/VMAS. The dedicated SLB would be created once the service reconcile loop is done. Unlike the primary SLB, there is no default outbound rules/IPs for the non-primary SLBs. That means the SLB would be deleted once all the services referencing it are deleted.
+
+## Choose which SLB to use
+
+The service annotation `service.beta.kubernetes.io/azure-load-balancer-mode` will be respected as long as `enableMultipleStandardLoadBalancers=true` when using standard LB, and the usage is the same as it is in the basic LB clusters. Specifically, there are three selection mode: `default` to select the primary SLB; `__auto__` to select the SLB with minimum rules and `vmSetName` to select the dedicated SLB of that VMSS/VMAS.
+
+## Outbound Connections of non-primary VMSS/VMAS
+
+The outbound rules of the non-primary SLB are not managed by cloud provider azure. Instead, it should be managed by cluster provisioning tools. For now, there is no outbound configuration for the non-primary VMSS/VMAS, but we plan to support customized outbound configurations in AKS and CAPZ in the future.
+
+## Sharing the primary SLB with multiple VMSS/VMAS
+
+> This feature is supported since v1.21.0
+
+For each non-primary VMSS/VMAS, one can determine to use dedicated SLB or share the primary SLB. If the VMSS/VMAS names are in the cloud config `nodepoolsWithoutDedicatedSLB`, those would join the backend pool of the primary SLB while the others would remain to have dedicated SLBs. If the VMSS/VMAS supposed to share the primary SLB owns a dedicated SLB, the dedicated one would be deleted, and the VMSS/VMAS would be joint the primary SLB's backend pool.
diff --git a/vendor/modules.txt b/vendor/modules.txt
index e737f5654..00d140208 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -550,7 +550,7 @@ gopkg.in/yaml.v2
 # gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b => gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 ## explicit
 gopkg.in/yaml.v3
-# k8s.io/api v0.23.1 => k8s.io/api v0.23.0
+# k8s.io/api v0.23.0 => k8s.io/api v0.23.0
 ## explicit; go 1.16
 k8s.io/api/admission/v1
 k8s.io/api/admission/v1beta1
@@ -599,7 +599,7 @@ k8s.io/api/scheduling/v1beta1
 k8s.io/api/storage/v1
 k8s.io/api/storage/v1alpha1
 k8s.io/api/storage/v1beta1
-# k8s.io/apimachinery v0.23.1 => k8s.io/apimachinery v0.23.0
+# k8s.io/apimachinery v0.23.0 => k8s.io/apimachinery v0.23.0
 ## explicit; go 1.16
 k8s.io/apimachinery/pkg/api/equality
 k8s.io/apimachinery/pkg/api/errors
@@ -784,7 +784,7 @@ k8s.io/apiserver/plugin/pkg/audit/truncate
 k8s.io/apiserver/plugin/pkg/audit/webhook
 k8s.io/apiserver/plugin/pkg/authenticator/token/webhook
 k8s.io/apiserver/plugin/pkg/authorizer/webhook
-# k8s.io/client-go v0.23.1 => k8s.io/client-go v0.23.0
+# k8s.io/client-go v0.23.0 => k8s.io/client-go v0.23.0
 ## explicit; go 1.16
 k8s.io/client-go/applyconfigurations/admissionregistration/v1
 k8s.io/client-go/applyconfigurations/admissionregistration/v1beta1
@@ -1107,7 +1107,7 @@ k8s.io/component-base/metrics/testutil
 k8s.io/component-base/term
 k8s.io/component-base/traces
 k8s.io/component-base/version
-# k8s.io/component-helpers v0.23.1
+# k8s.io/component-helpers v0.23.0
 ## explicit; go 1.16
 k8s.io/component-helpers/node/util
 # k8s.io/controller-manager v0.23.0 => k8s.io/controller-manager v0.23.0


Test patch:
diff --git a/pkg/azureclients/armclient/azure_armclient_test.go b/pkg/azureclients/armclient/azure_armclient_test.go
index 810dfb377..06e786011 100644
--- a/pkg/azureclients/armclient/azure_armclient_test.go
+++ b/pkg/azureclients/armclient/azure_armclient_test.go
@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"net/http/httptest"
+	"sync"
 	"testing"
 	"time"
 
@@ -343,6 +344,26 @@ func TestPutResource(t *testing.T) {
 }
 
 func TestPutResources(t *testing.T) {
+	total := 0
+	server := getTestServer(t, &total)
+
+	azConfig := azureclients.ClientConfig{Backoff: &retry.Backoff{Steps: 1}, UserAgent: "test", Location: "eastus"}
+	armClient := New(nil, azConfig, server.URL, "2019-01-01")
+	armClient.client.RetryDuration = time.Millisecond * 1
+
+	ctx := context.Background()
+	resources := map[string]interface{}{
+		"/id/1": nil,
+		"/id/2": nil,
+	}
+	responses := armClient.PutResources(ctx, nil)
+	assert.Nil(t, responses)
+	responses = armClient.PutResources(ctx, resources)
+	assert.NotNil(t, responses)
+	assert.Equal(t, 3, total)
+}
+
+func getTestServer(t *testing.T, counter *int) *httptest.Server {
 	serverFuncs := []func(rw http.ResponseWriter, req *http.Request){
 		func(rw http.ResponseWriter, req *http.Request) {
 			assert.Equal(t, "PUT", req.Method)
@@ -372,30 +393,81 @@ func TestPutResources(t *testing.T) {
 		},
 	}
 
-	i, total := 0, 0
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	i := 0
+	var l sync.Mutex
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		l.Lock()
 		serverFuncs[i](w, r)
 		i++
 		if i > 3 {
 			i = 3
 		}
-		total++
+		*counter++
+		l.Unlock()
 	}))
+}
 
-	azConfig := azureclients.ClientConfig{Backoff: &retry.Backoff{Steps: 1}, UserAgent: "test", Location: "eastus"}
-	armClient := New(nil, azConfig, server.URL, "2019-01-01")
-	armClient.client.RetryDuration = time.Millisecond * 1
-
-	ctx := context.Background()
-	resources := map[string]interface{}{
-		"/id/1": nil,
-		"/id/2": nil,
+func TestPutResourcesInBatches(t *testing.T) {
+	for _, testCase := range []struct {
+		description                  string
+		resources                    map[string]interface{}
+		batchSize, expectedCallTimes int
+	}{
+		{
+			description: "",
+			resources: map[string]interface{}{
+				"/id/1": nil,
+				"/id/2": nil,
+			},
+			batchSize:         2,
+			expectedCallTimes: 3,
+		},
+		{
+			description: "",
+			resources: map[string]interface{}{
+				"/id/1": nil,
+				"/id/2": nil,
+			},
+			batchSize:         1,
+			expectedCallTimes: 3,
+		},
+		{
+			description: "",
+			resources:   nil,
+		},
+		{
+			description: "PutResourcesInBatches should set the batch size to the length of the resources if the batch size is larger than it",
+			resources: map[string]interface{}{
+				"/id/1": nil,
+				"/id/2": nil,
+			},
+			batchSize:         10,
+			expectedCallTimes: 3,
+		},
+		{
+			description: "PutResourcesInBatches should call PutResources if the batch size is smaller than or equal to zero",
+			resources: map[string]interface{}{
+				"/id/1": nil,
+				"/id/2": nil,
+			},
+			expectedCallTimes: 3,
+		},
+	} {
+		t.Run(testCase.description, func(t *testing.T) {
+			total := 0
+			server := getTestServer(t, &total)
+
+			azConfig := azureclients.ClientConfig{Backoff: &retry.Backoff{Steps: 1}, UserAgent: "test", Location: "eastus"}
+			armClient := New(nil, azConfig, server.URL, "2019-01-01")
+			armClient.client.RetryDuration = time.Millisecond * 1
+
+			ctx := context.Background()
+			responses := armClient.PutResourcesInBatches(ctx, testCase.resources, testCase.batchSize)
+			assert.Equal(t, testCase.resources == nil, responses == nil)
+			assert.Equal(t, testCase.expectedCallTimes, total)
+		})
 	}
-	responses := armClient.PutResources(ctx, nil)
-	assert.Nil(t, responses)
-	responses = armClient.PutResources(ctx, resources)
-	assert.NotNil(t, responses)
-	assert.Equal(t, 3, total)
+
 }
 
 func TestPutResourceAsync(t *testing.T) {
diff --git a/pkg/azureclients/vmssvmclient/azure_vmssvmclient_test.go b/pkg/azureclients/vmssvmclient/azure_vmssvmclient_test.go
index 3a0406204..b30bfce7d 100644
--- a/pkg/azureclients/vmssvmclient/azure_vmssvmclient_test.go
+++ b/pkg/azureclients/vmssvmclient/azure_vmssvmclient_test.go
@@ -650,11 +650,11 @@ func TestUpdateVMs(t *testing.T) {
 	}
 
 	armClient := mockarmclient.NewMockInterface(ctrl)
-	armClient.EXPECT().PutResources(gomock.Any(), testvmssVMs).Return(responses).Times(1)
+	armClient.EXPECT().PutResourcesInBatches(gomock.Any(), testvmssVMs, 0).Return(responses).Times(1)
 	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(2)
 
 	vmssvmClient := getTestVMSSVMClient(armClient)
-	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
+	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
 	assert.Nil(t, rerr)
 }
 
@@ -679,11 +679,11 @@ func TestUpdateVMsWithUpdateVMsResponderError(t *testing.T) {
 		},
 	}
 	armClient := mockarmclient.NewMockInterface(ctrl)
-	armClient.EXPECT().PutResources(gomock.Any(), testvmssVMs).Return(responses).Times(1)
+	armClient.EXPECT().PutResourcesInBatches(gomock.Any(), testvmssVMs, 0).Return(responses).Times(1)
 	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
 
 	vmssvmClient := getTestVMSSVMClient(armClient)
-	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
+	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
 	assert.NotNil(t, rerr)
 }
 
@@ -699,7 +699,7 @@ func TestUpdateVMsNeverRateLimiter(t *testing.T) {
 
 	armClient := mockarmclient.NewMockInterface(ctrl)
 	vmssvmClient := getTestVMSSVMClientWithNeverRateLimiter(armClient)
-	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
+	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
 	assert.NotNil(t, rerr)
 	assert.Equal(t, vmssvmUpdateVMsErr, rerr)
 }
@@ -717,7 +717,7 @@ func TestUpdateVMsRetryAfterReader(t *testing.T) {
 
 	armClient := mockarmclient.NewMockInterface(ctrl)
 	vmClient := getTestVMSSVMClientWithRetryAfterReader(armClient)
-	rerr := vmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
+	rerr := vmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
 	assert.NotNil(t, rerr)
 	assert.Equal(t, vmssvmUpdateVMsErr, rerr)
 }
@@ -750,11 +750,11 @@ func TestUpdateVMsThrottle(t *testing.T) {
 	}
 
 	armClient := mockarmclient.NewMockInterface(ctrl)
-	armClient.EXPECT().PutResources(gomock.Any(), testvmssVMs).Return(responses).Times(1)
+	armClient.EXPECT().PutResourcesInBatches(gomock.Any(), testvmssVMs, 0).Return(responses).Times(1)
 	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
 
 	vmssvmClient := getTestVMSSVMClient(armClient)
-	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
+	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
 	assert.NotNil(t, rerr)
 	assert.EqualError(t, throttleErr.Error(), rerr.RawError.Error())
 }
diff --git a/pkg/provider/azure_vmss_test.go b/pkg/provider/azure_vmss_test.go
index 7c0349767..62af858be 100644
--- a/pkg/provider/azure_vmss_test.go
+++ b/pkg/provider/azure_vmss_test.go
@@ -2190,7 +2190,7 @@ func TestEnsureHostsInPool(t *testing.T) {
 		expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "", false)
 		mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 		mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
-		mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any()).Return(nil).Times(test.expectedVMSSVMPutTimes)
+		mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(test.expectedVMSSVMPutTimes)
 
 		mockVMClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
 		mockVMClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
@@ -2481,7 +2481,7 @@ func TestEnsureBackendPoolDeleted(t *testing.T) {
 		expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "", false)
 		mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 		mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
-		mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any()).Return(test.vmClientErr).Times(test.expectedVMSSVMPutTimes)
+		mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any(), gomock.Any()).Return(test.vmClientErr).Times(test.expectedVMSSVMPutTimes)
 
 		err = ss.EnsureBackendPoolDeleted(&v1.Service{}, test.backendpoolID, testVMSSName, test.backendAddressPools, true)
 		assert.Equal(t, test.expectedErr, err != nil, test.description+", but an error occurs")
@@ -2554,7 +2554,7 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) {
 	mockVMSSVMClient.EXPECT().List(gomock.Any(), "rg1", "vmss-0", gomock.Any()).Return(nil, nil).AnyTimes()
 	mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-0", gomock.Any()).Return(expectedVMSSVMsOfVMSS0, nil).AnyTimes()
 	mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-1", gomock.Any()).Return(expectedVMSSVMsOfVMSS1, nil).AnyTimes()
-	mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)
+	mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)
 
 	backendpoolAddressIDs := []string{testLBBackendpoolID0, testLBBackendpoolID1, testLBBackendpoolID2}
 	testVMSSNames := []string{"vmss-0", "vmss-1", "vmss-2"}