Repository: kubernetes-sigs/cloud-provider-azure
Pull request #981 (2022-01-08 03:16:45)

cherry-pick #926 #936 #978 to release-1.0
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?

<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

#### What this PR does / why we need it:
refer #926 #936 #978
#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
NONE
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```



Patch:
diff --git a/Dockerfile b/Dockerfile
index 3ea9fa9e9..472b515a8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,8 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.16.6-stretch AS builder
+FROM --platform=linux/amd64 golang:1.16.6-stretch AS builder
+
 ARG ENABLE_GIT_COMMAND=true
+ARG ARCH=amd64
+
 WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure
 COPY . .
 
@@ -24,8 +27,8 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     go build ./cmd/cloud-controller-manager
 
 RUN --mount=type=cache,target=/root/.cache/go-build \
-    make bin/azure-cloud-controller-manager ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND}
+     make bin/azure-cloud-controller-manager ENABLE_GIT_COMMAND=${ENABLE_GIT_COMMAND} ARCH=${ARCH}
 
 FROM gcr.io/distroless/static
 COPY --from=builder /go/src/sigs.k8s.io/cloud-provider-azure/bin/azure-cloud-controller-manager /usr/local/bin/cloud-controller-manager
-ENTRYPOINT [ "/usr/local/bin/cloud-controller-manager" ]
\ No newline at end of file
+ENTRYPOINT [ "/usr/local/bin/cloud-controller-manager" ]
diff --git a/Makefile b/Makefile
index edea35ec1..70dc13e1e 100644
--- a/Makefile
+++ b/Makefile
@@ -62,7 +62,11 @@ endif
 DOCKER_CLI_EXPERIMENTAL := enabled
 
 # cloud controller manager image
+ifeq ($(ARCH), amd64)
 IMAGE_NAME=azure-cloud-controller-manager
+else
+IMAGE_NAME=azure-cloud-controller-manager-$(ARCH)
+endif
 IMAGE=$(IMAGE_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG)
 # cloud node manager image
 NODE_MANAGER_IMAGE_NAME=azure-cloud-node-manager
@@ -101,7 +105,7 @@ $(BIN_DIR)/azure-cloud-node-manager.exe: $(PKG_CONFIG) $(wildcard cmd/cloud-node
 	CGO_ENABLED=0 GOOS=windows go build -a -o $(BIN_DIR)/azure-cloud-node-manager.exe $(shell cat $(PKG_CONFIG)) ./cmd/cloud-node-manager
 
 $(BIN_DIR)/azure-cloud-controller-manager: $(PKG_CONFIG) $(wildcard cmd/cloud-controller-manager/*) $(wildcard cmd/cloud-controller-manager/**/*) $(wildcard pkg/**/*) ## Build binary for controller-manager.
-	CGO_ENABLED=0 GOOS=linux go build -a -o $(BIN_DIR)/azure-cloud-controller-manager $(shell cat $(PKG_CONFIG)) ./cmd/cloud-controller-manager
+	CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -a -o $(BIN_DIR)/azure-cloud-controller-manager $(shell cat $(PKG_CONFIG)) ./cmd/cloud-controller-manager
 
 ## --------------------------------------
 ##@ Images
@@ -121,8 +125,16 @@ buildx-setup:
 	docker run --rm --privileged tonistiigi/binfmt --install all
 
 .PHONY: build-ccm-image
-build-ccm-image: docker-pull-prerequisites ## Build controller-manager image.
-	DOCKER_BUILDKIT=1 docker build -t $(IMAGE) --build-arg ENABLE_GIT_COMMAND=$(ENABLE_GIT_COMMAND) .
+build-ccm-image: buildx-setup docker-pull-prerequisites ## Build controller-manager image.
+	docker buildx build \
+		--pull \
+		--output=type=$(OUTPUT_TYPE) \
+		--platform linux/$(ARCH) \
+		--build-arg ENABLE_GIT_COMMAND="$(ENABLE_GIT_COMMAND)" \
+		--build-arg ARCH="$(ARCH)" \
+		--build-arg VERSION="$(VERSION)" \
+		--file Dockerfile \
+		--tag $(IMAGE) .
 
 .PHONY: build-node-image
 build-node-image: buildx-setup docker-pull-prerequisites ## Build node-manager image.
@@ -171,16 +183,16 @@ endif
 ## --------------------------------------
 
 .PHONY: build-images
-build-images: build-ccm-image build-all-node-images ## Build all images.
+build-images: build-all-ccm-images build-all-node-images ## Build all images.
 
 .PHONY: image
-image: build-ccm-image build-all-node-images ## Build all images.
+image: build-all-ccm-images build-all-node-images ## Build all images.
 
 .PHONY: push-images
-push-images: push-ccm-image push-all-node-images ## Push all images.
+push-images: push-all-ccm-images push-all-node-images ## Push all images.
 
 .PHONY: push
-push: push-ccm-image push-all-node-images ## Push all images.
+push: push-all-ccm-images push-all-node-images ## Push all images.
 
 .PHONY: push-node-manager-manifest
 push-node-manager-manifest: push-all-node-images push-all-windows-node-images ## Create and push a manifest list containing all the Windows and Linux images.
@@ -223,6 +235,19 @@ push-node-image-windows-%: ## Push node-manager image for Windows.
 push-node-image-%:
 	$(MAKE) ARCH=$* push-node-image
 
+.PHONY: build-all-ccm-images
+build-all-ccm-images: $(addprefix build-ccm-image-,$(LINUX_ARCHS))
+
+build-ccm-image-%:
+	$(MAKE) ARCH=$* build-ccm-image
+
+.PHONY: push-all-ccm-images
+push-all-ccm-images: $(addprefix push-ccm-image-,$(LINUX_ARCHS))
+
+push-ccm-image-%:
+	$(MAKE) ARCH=$* push-ccm-image
+
+
 ## --------------------------------------
 ##@ Tests
 ## --------------------------------------
diff --git a/cloud-node-manager.Dockerfile b/cloud-node-manager.Dockerfile
index adaabdec7..dac1adf3a 100644
--- a/cloud-node-manager.Dockerfile
+++ b/cloud-node-manager.Dockerfile
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=linux/amd64 golang:1.16.6-stretch AS builder
+FROM golang:1.16.6-stretch AS builder
 
 ARG ENABLE_GIT_COMMAND=true
 ARG ARCH=amd64
diff --git a/pkg/azureclients/armclient/azure_armclient.go b/pkg/azureclients/armclient/azure_armclient.go
index 90f2a34d4..72d4b8942 100644
--- a/pkg/azureclients/armclient/azure_armclient.go
+++ b/pkg/azureclients/armclient/azure_armclient.go
@@ -361,7 +361,46 @@ func (c *Client) PutResource(ctx context.Context, resourceID string, parameters
 	return c.PutResourceWithDecorators(ctx, resourceID, parameters, putDecorators)
 }
 
-func (c *Client) waitAsync(ctx context.Context, futures map[string]*azure.Future, previousResponses map[string]*PutResourcesResponse) {
+// PutResources puts a list of resources from resources map[resourceID]parameters.
+// Those resources sync requests are sequential while async requests are concurrent. It's especially
+// useful when the ARM API doesn't support concurrent requests.
+func (c *Client) PutResources(ctx context.Context, resources map[string]interface{}) map[string]*PutResourcesResponse {
+	if len(resources) == 0 {
+		return nil
+	}
+
+	// Sequential sync requests.
+	futures := make(map[string]*azure.Future)
+	responses := make(map[string]*PutResourcesResponse)
+	for resourceID, parameters := range resources {
+		decorators := []autorest.PrepareDecorator{
+			autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
+			autorest.WithJSON(parameters),
+		}
+		request, err := c.PreparePutRequest(ctx, decorators...)
+		if err != nil {
+			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.prepare", resourceID, err)
+			responses[resourceID] = &PutResourcesResponse{
+				Error: retry.NewError(false, err),
+			}
+			continue
+		}
+		dumpRequest(request, 10)
+
+		future, resp, clientErr := c.SendAsync(ctx, request)
+		defer c.CloseResponse(ctx, resp)
+		if clientErr != nil {
+			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "put.send", resourceID, clientErr.Error())
+			responses[resourceID] = &PutResourcesResponse{
+				Error: clientErr,
+			}
+			continue
+		}
+
+		futures[resourceID] = future
+	}
+
+	// Concurrent async requests.
 	wg := sync.WaitGroup{}
 	var responseLock sync.Mutex
 	for resourceID, future := range futures {
@@ -384,111 +423,22 @@ func (c *Client) waitAsync(ctx context.Context, futures map[string]*azure.Future
 				}
 
 				responseLock.Lock()
-				previousResponses[resourceID] = &PutResourcesResponse{
+				responses[resourceID] = &PutResourcesResponse{
 					Error: retriableErr,
 				}
 				responseLock.Unlock()
 				return
 			}
-		}(resourceID, future)
-	}
-	wg.Wait()
-}
-
-// PutResources puts a list of resources from resources map[resourceID]parameters.
-// Those resources sync requests are sequential while async requests are concurrent. It's especially
-// useful when the ARM API doesn't support concurrent requests.
-func (c *Client) PutResources(ctx context.Context, resources map[string]interface{}) map[string]*PutResourcesResponse {
-	if len(resources) == 0 {
-		return nil
-	}
 
-	// Sequential sync requests.
-	futures := make(map[string]*azure.Future)
-	responses := make(map[string]*PutResourcesResponse)
-	for resourceID, parameters := range resources {
-		future, rerr := c.PutResourceAsync(ctx, resourceID, parameters)
-		if rerr != nil {
+			responseLock.Lock()
 			responses[resourceID] = &PutResourcesResponse{
-				Error: rerr,
+				Response: response,
 			}
-			continue
-		}
-		futures[resourceID] = future
-	}
-
-	c.waitAsync(ctx, futures, responses)
-
-	return responses
-}
-
-// PutResourcesInBatches is similar with PutResources, but it sends sync request concurrently in batches.
-func (c *Client) PutResourcesInBatches(ctx context.Context, resources map[string]interface{}, batchSize int) map[string]*PutResourcesResponse {
-	if len(resources) == 0 {
-		return nil
-	}
-
-	if batchSize <= 0 {
-		klog.V(4).Infof("PutResourcesInBatches: batch size %d, put resources in sequence", batchSize)
-		return c.PutResources(ctx, resources)
-	}
-
-	if batchSize > len(resources) {
-		klog.V(4).Infof("PutResourcesInBatches: batch size %d, but the number of the resources is %d", batchSize, resources)
-		batchSize = len(resources)
-	}
-	klog.V(4).Infof("PutResourcesInBatches: send sync requests in parallel with the batch size %d", batchSize)
-
-	// Convert map to slice because it is more straightforward to
-	// loop over slice in batches than map.
-	type resourcesMeta struct {
-		resourceID string
-		parameters interface{}
-	}
-	resourcesList := make([]resourcesMeta, 0)
-	for resourceID, parameters := range resources {
-		resourcesList = append(resourcesList, resourcesMeta{
-			resourceID: resourceID,
-			parameters: parameters,
-		})
-	}
-
-	// Concurrent sync requests in batches.
-	futures := make(map[string]*azure.Future)
-	responses := make(map[string]*PutResourcesResponse)
-	wg := sync.WaitGroup{}
-	var responseLock, futuresLock sync.Mutex
-	for i := 0; i < len(resourcesList); i += batchSize {
-		j := i + batchSize
-		if j > len(resourcesList) {
-			j = len(resourcesList)
-		}
-
-		for k := i; k < j; k++ {
-			wg.Add(1)
-			go func(resourceID string, parameters interface{}) {
-				defer wg.Done()
-				future, rerr := c.PutResourceAsync(ctx, resourceID, parameters)
-				if rerr != nil {
-					responseLock.Lock()
-					responses[resourceID] = &PutResourcesResponse{
-						Error: rerr,
-					}
-					responseLock.Unlock()
-					return
-				}
-
-				futuresLock.Lock()
-				futures[resourceID] = future
-				futuresLock.Unlock()
-			}(resourcesList[k].resourceID, resourcesList[k].parameters)
-		}
-		wg.Wait()
+			responseLock.Unlock()
+		}(resourceID, future)
 	}
 
-	// Concurrent async requests.
-	c.waitAsync(ctx, futures, responses)
-
+	wg.Wait()
 	return responses
 }
 
diff --git a/pkg/azureclients/armclient/interface.go b/pkg/azureclients/armclient/interface.go
index 0926c903b..e7d437e3e 100644
--- a/pkg/azureclients/armclient/interface.go
+++ b/pkg/azureclients/armclient/interface.go
@@ -71,9 +71,6 @@ type Interface interface {
 	// useful when the ARM API doesn't support concurrent requests.
 	PutResources(ctx context.Context, resources map[string]interface{}) map[string]*PutResourcesResponse
 
-	// PutResourcesInBatches is similar with PutResources, but it sends sync request concurrently in batches.
-	PutResourcesInBatches(ctx context.Context, resources map[string]interface{}, batchSize int) map[string]*PutResourcesResponse
-
 	// PutResourceWithDecorators puts a resource with decorators by resource ID
 	PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error)
 
diff --git a/pkg/azureclients/armclient/mockarmclient/interface.go b/pkg/azureclients/armclient/mockarmclient/interface.go
index 5802fa8a7..bd1246839 100644
--- a/pkg/azureclients/armclient/mockarmclient/interface.go
+++ b/pkg/azureclients/armclient/mockarmclient/interface.go
@@ -240,20 +240,6 @@ func (mr *MockInterfaceMockRecorder) PutResources(ctx, resources interface{}) *g
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResources", reflect.TypeOf((*MockInterface)(nil).PutResources), ctx, resources)
 }
 
-// PutResourcesInBatches mocks base method
-func (m *MockInterface) PutResourcesInBatches(ctx context.Context, resources map[string]interface{}, batchSize int) map[string]*armclient.PutResourcesResponse {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "PutResourcesInBatches", ctx, resources, batchSize)
-	ret0, _ := ret[0].(map[string]*armclient.PutResourcesResponse)
-	return ret0
-}
-
-// PutResourcesInBatches indicates an expected call of PutResourcesInBatches
-func (mr *MockInterfaceMockRecorder) PutResourcesInBatches(ctx, resources, batchSize interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResourcesInBatches", reflect.TypeOf((*MockInterface)(nil).PutResourcesInBatches), ctx, resources, batchSize)
-}
-
 // PutResourceWithDecorators mocks base method
 func (m *MockInterface) PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
 	m.ctrl.T.Helper()
diff --git a/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go b/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go
index 8205f336b..a8716c6af 100644
--- a/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go
+++ b/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go
@@ -384,9 +384,7 @@ func (page VirtualMachineScaleSetVMListResultPage) Values() []compute.VirtualMac
 }
 
 // UpdateVMs updates a list of VirtualMachineScaleSetVM from map[instanceID]compute.VirtualMachineScaleSetVM.
-// If the batch size > 0, it will send sync requests concurrently in batches, or it will send sync requests in sequence.
-// No matter what the batch size is, it will process the async requests concurrently in one single batch.
-func (c *Client) UpdateVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string, batchSize int) *retry.Error {
+func (c *Client) UpdateVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string) *retry.Error {
 	mc := metrics.NewMetricContext("vmssvm", "update_vms", resourceGroupName, c.subscriptionID, source)
 
 	// Report errors if the client is rate limited.
@@ -402,7 +400,7 @@ func (c *Client) UpdateVMs(ctx context.Context, resourceGroupName string, VMScal
 		return rerr
 	}
 
-	rerr := c.updateVMSSVMs(ctx, resourceGroupName, VMScaleSetName, instances, batchSize)
+	rerr := c.updateVMSSVMs(ctx, resourceGroupName, VMScaleSetName, instances)
 	_ = mc.Observe(rerr.Error())
 	if rerr != nil {
 		if rerr.IsThrottled() {
@@ -417,7 +415,7 @@ func (c *Client) UpdateVMs(ctx context.Context, resourceGroupName string, VMScal
 }
 
 // updateVMSSVMs updates a list of VirtualMachineScaleSetVM from map[instanceID]compute.VirtualMachineScaleSetVM.
-func (c *Client) updateVMSSVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, batchSize int) *retry.Error {
+func (c *Client) updateVMSSVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM) *retry.Error {
 	resources := make(map[string]interface{})
 	for instanceID, parameter := range instances {
 		resourceID := armclient.GetChildResourceID(
@@ -431,7 +429,7 @@ func (c *Client) updateVMSSVMs(ctx context.Context, resourceGroupName string, VM
 		resources[resourceID] = parameter
 	}
 
-	responses := c.armClient.PutResourcesInBatches(ctx, resources, batchSize)
+	responses := c.armClient.PutResources(ctx, resources)
 	errors := make([]*retry.Error, 0)
 	for resourceID, resp := range responses {
 		if resp == nil {
diff --git a/pkg/azureclients/vmssvmclient/interface.go b/pkg/azureclients/vmssvmclient/interface.go
index 9743a9a69..165e33215 100644
--- a/pkg/azureclients/vmssvmclient/interface.go
+++ b/pkg/azureclients/vmssvmclient/interface.go
@@ -47,5 +47,5 @@ type Interface interface {
 	Update(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string, parameters compute.VirtualMachineScaleSetVM, source string) *retry.Error
 
 	// UpdateVMs updates a list of VirtualMachineScaleSetVM from map[instanceID]compute.VirtualMachineScaleSetVM.
-	UpdateVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string, batchSize int) *retry.Error
+	UpdateVMs(ctx context.Context, resourceGroupName string, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string) *retry.Error
 }
diff --git a/pkg/azureclients/vmssvmclient/mockvmssvmclient/interface.go b/pkg/azureclients/vmssvmclient/mockvmssvmclient/interface.go
index cbbc21aff..30d240d30 100644
--- a/pkg/azureclients/vmssvmclient/mockvmssvmclient/interface.go
+++ b/pkg/azureclients/vmssvmclient/mockvmssvmclient/interface.go
@@ -93,15 +93,15 @@ func (mr *MockInterfaceMockRecorder) Update(ctx, resourceGroupName, VMScaleSetNa
 }
 
 // UpdateVMs mocks base method
-func (m *MockInterface) UpdateVMs(ctx context.Context, resourceGroupName, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string, batchSize int) *retry.Error {
+func (m *MockInterface) UpdateVMs(ctx context.Context, resourceGroupName, VMScaleSetName string, instances map[string]compute.VirtualMachineScaleSetVM, source string) *retry.Error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "UpdateVMs", ctx, resourceGroupName, VMScaleSetName, instances, source, batchSize)
+	ret := m.ctrl.Call(m, "UpdateVMs", ctx, resourceGroupName, VMScaleSetName, instances, source)
 	ret0, _ := ret[0].(*retry.Error)
 	return ret0
 }
 
 // UpdateVMs indicates an expected call of UpdateVMs
-func (mr *MockInterfaceMockRecorder) UpdateVMs(ctx, resourceGroupName, VMScaleSetName, instances, source, batchSize interface{}) *gomock.Call {
+func (mr *MockInterfaceMockRecorder) UpdateVMs(ctx, resourceGroupName, VMScaleSetName, instances, source interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVMs", reflect.TypeOf((*MockInterface)(nil).UpdateVMs), ctx, resourceGroupName, VMScaleSetName, instances, source, batchSize)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVMs", reflect.TypeOf((*MockInterface)(nil).UpdateVMs), ctx, resourceGroupName, VMScaleSetName, instances, source)
 }
diff --git a/pkg/provider/azure.go b/pkg/provider/azure.go
index 443314f6c..d598c94fc 100644
--- a/pkg/provider/azure.go
+++ b/pkg/provider/azure.go
@@ -225,9 +225,6 @@ type Config struct {
 	// RouteUpdateWaitingInSeconds is the delay time for waiting route updates to take effect. This waiting delay is added
 	// because the routes are not taken effect when the async route updating operation returns success. Default is 30 seconds.
 	RouteUpdateWaitingInSeconds int `json:"routeUpdateWaitingInSeconds,omitempty" yaml:"routeUpdateWaitingInSeconds,omitempty"`
-	// PutVMSSVMBatchSize defines how many requests the client send concurrently when putting the VMSS VMs.
-	// If it is smaller than or equal to zero, the request will be sent one by one in sequence (default).
-	PutVMSSVMBatchSize int `json:"putVMSSVMBatchSize" yaml:"putVMSSVMBatchSize"`
 }
 
 type InitSecretConfig struct {
@@ -573,10 +570,6 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret, syncZones
 	return nil
 }
 
-func (az *Cloud) getPutVMSSVMBatchSize() int {
-	return az.PutVMSSVMBatchSize
-}
-
 func (az *Cloud) initCaches() (err error) {
 	az.vmCache, err = az.newVMCache()
 	if err != nil {
diff --git a/pkg/provider/azure_backoff.go b/pkg/provider/azure_backoff.go
index 9558d0ab5..d66dbc6c6 100644
--- a/pkg/provider/azure_backoff.go
+++ b/pkg/provider/azure_backoff.go
@@ -271,11 +271,6 @@ func (az *Cloud) ListManagedLBs(service *v1.Service, nodes []*v1.Node, clusterNa
 		return nil, nil
 	}
 
-	// return early if wantLb=false
-	if nodes == nil {
-		return allLBs, nil
-	}
-
 	agentPoolLBs := make([]network.LoadBalancer, 0)
 	agentPoolVMSetNames, err := az.VMSet.GetAgentPoolVMSetNames(nodes)
 	if err != nil {
@@ -290,10 +285,8 @@ func (az *Cloud) ListManagedLBs(service *v1.Service, nodes []*v1.Node, clusterNa
 
 	for _, lb := range allLBs {
 		vmSetNameFromLBName := az.mapLoadBalancerNameToVMSet(to.String(lb.Name), clusterName)
-		if strings.EqualFold(strings.TrimSuffix(to.String(lb.Name), consts.InternalLoadBalancerNameSuffix), clusterName) ||
-			agentPoolVMSetNamesSet.Has(strings.ToLower(vmSetNameFromLBName)) {
+		if agentPoolVMSetNamesSet.Has(strings.ToLower(vmSetNameFromLBName)) {
 			agentPoolLBs = append(agentPoolLBs, lb)
-			klog.V(4).Infof("ListManagedLBs: found agent pool LB %s", to.String(lb.Name))
 		}
 	}
 
diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go
index 9fe42198c..68224ba4a 100644
--- a/pkg/provider/azure_vmss.go
+++ b/pkg/provider/azure_vmss.go
@@ -1319,7 +1319,7 @@ func (ss *ScaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac
 			ctx, cancel := getContextWithCancel()
 			defer cancel()
 			klog.V(2).Infof("EnsureHostInPool begins to UpdateVMs for VMSS(%s, %s) with new backendPoolID %s", meta.resourceGroup, meta.vmssName, backendPoolID)
-			rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update", ss.getPutVMSSVMBatchSize())
+			rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update")
 			if rerr != nil {
 				klog.Errorf("EnsureHostInPool UpdateVMs for VMSS(%s, %s) failed with error %v", meta.resourceGroup, meta.vmssName, rerr.Error())
 				return rerr.Error()
@@ -1581,7 +1581,7 @@ func (ss *ScaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID,
 			ctx, cancel := getContextWithCancel()
 			defer cancel()
 			klog.V(2).Infof("EnsureBackendPoolDeleted begins to UpdateVMs for VMSS(%s, %s) with backendPoolID %s", meta.resourceGroup, meta.vmssName, backendPoolID)
-			rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update", ss.getPutVMSSVMBatchSize())
+			rerr := ss.VirtualMachineScaleSetVMsClient.UpdateVMs(ctx, meta.resourceGroup, meta.vmssName, update, "network_update")
 			if rerr != nil {
 				klog.Errorf("EnsureBackendPoolDeleted UpdateVMs for VMSS(%s, %s) failed with error %v", meta.resourceGroup, meta.vmssName, rerr.Error())
 				return rerr.Error()


Test patch:
diff --git a/pkg/azureclients/armclient/azure_armclient_test.go b/pkg/azureclients/armclient/azure_armclient_test.go
index 2fc0dfd08..c9d80bc0c 100644
--- a/pkg/azureclients/armclient/azure_armclient_test.go
+++ b/pkg/azureclients/armclient/azure_armclient_test.go
@@ -22,7 +22,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"sync"
 	"testing"
 	"time"
 
@@ -341,26 +340,6 @@ func TestPutResource(t *testing.T) {
 }
 
 func TestPutResources(t *testing.T) {
-	total := 0
-	server := getTestServer(t, &total)
-
-	backoff := &retry.Backoff{Steps: 1}
-	armClient := New(nil, server.URL, "test", "2019-01-01", "eastus", backoff)
-	armClient.client.RetryDuration = time.Millisecond * 1
-
-	ctx := context.Background()
-	resources := map[string]interface{}{
-		"/id/1": nil,
-		"/id/2": nil,
-	}
-	responses := armClient.PutResources(ctx, nil)
-	assert.Nil(t, responses)
-	responses = armClient.PutResources(ctx, resources)
-	assert.NotNil(t, responses)
-	assert.Equal(t, 3, total)
-}
-
-func getTestServer(t *testing.T, counter *int) *httptest.Server {
 	serverFuncs := []func(rw http.ResponseWriter, req *http.Request){
 		func(rw http.ResponseWriter, req *http.Request) {
 			assert.Equal(t, "PUT", req.Method)
@@ -390,81 +369,30 @@ func getTestServer(t *testing.T, counter *int) *httptest.Server {
 		},
 	}
 
-	i := 0
-	var l sync.Mutex
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		l.Lock()
+	i, total := 0, 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		serverFuncs[i](w, r)
 		i++
 		if i > 3 {
 			i = 3
 		}
-		*counter++
-		l.Unlock()
+		total++
 	}))
-}
 
-func TestPutResourcesInBatches(t *testing.T) {
-	for _, testCase := range []struct {
-		description                  string
-		resources                    map[string]interface{}
-		batchSize, expectedCallTimes int
-	}{
-		{
-			description: "",
-			resources: map[string]interface{}{
-				"/id/1": nil,
-				"/id/2": nil,
-			},
-			batchSize:         2,
-			expectedCallTimes: 3,
-		},
-		{
-			description: "",
-			resources: map[string]interface{}{
-				"/id/1": nil,
-				"/id/2": nil,
-			},
-			batchSize:         1,
-			expectedCallTimes: 3,
-		},
-		{
-			description: "",
-			resources:   nil,
-		},
-		{
-			description: "PutResourcesInBatches should set the batch size to the length of the resources if the batch size is larger than it",
-			resources: map[string]interface{}{
-				"/id/1": nil,
-				"/id/2": nil,
-			},
-			batchSize:         10,
-			expectedCallTimes: 3,
-		},
-		{
-			description: "PutResourcesInBatches should call PutResources if the batch size is smaller than or equal to zero",
-			resources: map[string]interface{}{
-				"/id/1": nil,
-				"/id/2": nil,
-			},
-			expectedCallTimes: 3,
-		},
-	} {
-		t.Run(testCase.description, func(t *testing.T) {
-			total := 0
-			server := getTestServer(t, &total)
-
-			backoff := &retry.Backoff{Steps: 1}
-			armClient := New(nil, server.URL, "test", "2019-01-01", "eastus", backoff)
-			armClient.client.RetryDuration = time.Millisecond * 1
-
-			ctx := context.Background()
-			responses := armClient.PutResourcesInBatches(ctx, testCase.resources, testCase.batchSize)
-			assert.Equal(t, testCase.resources == nil, responses == nil)
-			assert.Equal(t, testCase.expectedCallTimes, total)
-		})
-	}
+	backoff := &retry.Backoff{Steps: 1}
+	armClient := New(nil, server.URL, "test", "2019-01-01", "eastus", backoff)
+	armClient.client.RetryDuration = time.Millisecond * 1
 
+	ctx := context.Background()
+	resources := map[string]interface{}{
+		"/id/1": nil,
+		"/id/2": nil,
+	}
+	responses := armClient.PutResources(ctx, nil)
+	assert.Nil(t, responses)
+	responses = armClient.PutResources(ctx, resources)
+	assert.NotNil(t, responses)
+	assert.Equal(t, 3, total)
 }
 
 func TestPutResourceAsync(t *testing.T) {
diff --git a/pkg/azureclients/vmssvmclient/azure_vmssvmclient_test.go b/pkg/azureclients/vmssvmclient/azure_vmssvmclient_test.go
index d59e912db..c1eb44132 100644
--- a/pkg/azureclients/vmssvmclient/azure_vmssvmclient_test.go
+++ b/pkg/azureclients/vmssvmclient/azure_vmssvmclient_test.go
@@ -619,11 +619,11 @@ func TestUpdateVMs(t *testing.T) {
 	}
 
 	armClient := mockarmclient.NewMockInterface(ctrl)
-	armClient.EXPECT().PutResourcesInBatches(gomock.Any(), testvmssVMs, 0).Return(responses).Times(1)
+	armClient.EXPECT().PutResources(gomock.Any(), testvmssVMs).Return(responses).Times(1)
 	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(2)
 
 	vmssvmClient := getTestVMSSVMClient(armClient)
-	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
+	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
 	assert.Nil(t, rerr)
 }
 
@@ -648,11 +648,11 @@ func TestUpdateVMsWithUpdateVMsResponderError(t *testing.T) {
 		},
 	}
 	armClient := mockarmclient.NewMockInterface(ctrl)
-	armClient.EXPECT().PutResourcesInBatches(gomock.Any(), testvmssVMs, 0).Return(responses).Times(1)
+	armClient.EXPECT().PutResources(gomock.Any(), testvmssVMs).Return(responses).Times(1)
 	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
 
 	vmssvmClient := getTestVMSSVMClient(armClient)
-	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
+	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
 	assert.NotNil(t, rerr)
 }
 
@@ -668,7 +668,7 @@ func TestUpdateVMsNeverRateLimiter(t *testing.T) {
 
 	armClient := mockarmclient.NewMockInterface(ctrl)
 	vmssvmClient := getTestVMSSVMClientWithNeverRateLimiter(armClient)
-	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
+	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
 	assert.NotNil(t, rerr)
 	assert.Equal(t, vmssvmUpdateVMsErr, rerr)
 }
@@ -686,7 +686,7 @@ func TestUpdateVMsRetryAfterReader(t *testing.T) {
 
 	armClient := mockarmclient.NewMockInterface(ctrl)
 	vmClient := getTestVMSSVMClientWithRetryAfterReader(armClient)
-	rerr := vmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
+	rerr := vmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
 	assert.NotNil(t, rerr)
 	assert.Equal(t, vmssvmUpdateVMsErr, rerr)
 }
@@ -719,11 +719,11 @@ func TestUpdateVMsThrottle(t *testing.T) {
 	}
 
 	armClient := mockarmclient.NewMockInterface(ctrl)
-	armClient.EXPECT().PutResourcesInBatches(gomock.Any(), testvmssVMs, 0).Return(responses).Times(1)
+	armClient.EXPECT().PutResources(gomock.Any(), testvmssVMs).Return(responses).Times(1)
 	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
 
 	vmssvmClient := getTestVMSSVMClient(armClient)
-	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test", 0)
+	rerr := vmssvmClient.UpdateVMs(context.TODO(), "rg", "vmss1", instances, "test")
 	assert.NotNil(t, rerr)
 	assert.EqualError(t, throttleErr.Error(), rerr.RawError.Error())
 }
diff --git a/pkg/provider/azure_vmss_test.go b/pkg/provider/azure_vmss_test.go
index 3fc0284b7..aa1ae8ce7 100644
--- a/pkg/provider/azure_vmss_test.go
+++ b/pkg/provider/azure_vmss_test.go
@@ -2190,7 +2190,7 @@ func TestEnsureHostsInPool(t *testing.T) {
 		expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "", false)
 		mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 		mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
-		mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(test.expectedVMSSVMPutTimes)
+		mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any()).Return(nil).Times(test.expectedVMSSVMPutTimes)
 
 		mockVMClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
 		mockVMClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
@@ -2481,7 +2481,7 @@ func TestEnsureBackendPoolDeleted(t *testing.T) {
 		expectedVMSSVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, []string{"vmss-vm-000000", "vmss-vm-000001", "vmss-vm-000002"}, "", false)
 		mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 		mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
-		mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any(), gomock.Any()).Return(test.vmClientErr).Times(test.expectedVMSSVMPutTimes)
+		mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any(), gomock.Any()).Return(test.vmClientErr).Times(test.expectedVMSSVMPutTimes)
 
 		err = ss.EnsureBackendPoolDeleted(&v1.Service{}, test.backendpoolID, testVMSSName, test.backendAddressPools, true)
 		assert.Equal(t, test.expectedErr, err != nil, test.description+", but an error occurs")
@@ -2554,7 +2554,7 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) {
 	mockVMSSVMClient.EXPECT().List(gomock.Any(), "rg1", "vmss-0", gomock.Any()).Return(nil, nil).AnyTimes()
 	mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-0", gomock.Any()).Return(expectedVMSSVMsOfVMSS0, nil).AnyTimes()
 	mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, "vmss-1", gomock.Any()).Return(expectedVMSSVMsOfVMSS1, nil).AnyTimes()
-	mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)
+	mockVMSSVMClient.EXPECT().UpdateVMs(gomock.Any(), ss.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(2)
 
 	backendpoolAddressIDs := []string{testLBBackendpoolID0, testLBBackendpoolID1, testLBBackendpoolID2}
 	testVMSSNames := []string{"vmss-0", "vmss-1", "vmss-2"}
diff --git a/tests/e2e/network/standard_lb.go b/tests/e2e/network/standard_lb.go
index 6da52058b..9b5da620a 100644
--- a/tests/e2e/network/standard_lb.go
+++ b/tests/e2e/network/standard_lb.go
@@ -161,9 +161,7 @@ var _ = Describe("[StandardLoadBalancer] Standard load balancer", func() {
 				}
 			}
 		}
-		if len(outboundRuleIPs) == 0 {
-			Skip("skip validating outbound IPs since outbound rules are not configured on SLB")
-		}
+		Expect(len(outboundRuleIPs)).NotTo(Equal(0))
 
 		podTemplate := createPodGetIP()
 		err = utils.CreatePod(cs, ns.Name, podTemplate)
diff --git a/tests/e2e/utils/network_utils.go b/tests/e2e/utils/network_utils.go
index 1e8c204d9..d4f482477 100644
--- a/tests/e2e/utils/network_utils.go
+++ b/tests/e2e/utils/network_utils.go
@@ -242,7 +242,7 @@ func SelectAvailablePrivateIP(tc *AzureTestClient) (string, error) {
 	if len(*vNet.Subnets) > 1 {
 		for _, sn := range *vNet.Subnets {
 			// if there is more than one subnet, select the first one we find.
-			if !strings.Contains(*sn.Name, "controlplane") && !strings.Contains(*sn.Name, "control-plane") {
+			if !strings.Contains(*sn.Name, "controlplane") {
 				subnet = *sn.AddressPrefix
 				break
 			}