Repository: kubernetes-sigs/cloud-provider-azure
Pull request #1716 (2022-05-13 07:47:46)

Cherrypick armclients 1.24
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?
/kind cleanup
<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

#### What this PR does / why we need it:
Cherrypick #1613 #1556
#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
NONE
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```



Patch:
diff --git a/pkg/azureclients/armclient/azure_armclient.go b/pkg/azureclients/armclient/azure_armclient.go
index d90e472b2..e4a403bb9 100644
--- a/pkg/azureclients/armclient/azure_armclient.go
+++ b/pkg/azureclients/armclient/azure_armclient.go
@@ -498,6 +498,15 @@ func (c *Client) PutResourcesInBatches(ctx context.Context, resources map[string
 
 // PutResourceWithDecorators puts a resource by resource ID
 func (c *Client) PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
+	return c.putResourceWithDecorators(ctx, resourceID, parameters, decorators, true)
+}
+
+// PutResourceWithDecoratorsAsync puts a resource by resource ID in async mode.
+func (c *Client) PutResourceWithDecoratorsAsync(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
+	return c.putResourceWithDecorators(ctx, resourceID, parameters, decorators, false)
+}
+
+func (c *Client) putResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator, wait bool) (*http.Response, *retry.Error) {
 	request, err := c.PreparePutRequest(ctx, decorators...)
 	dumpRequest(request, 10)
 	if err != nil {
@@ -512,6 +521,9 @@ func (c *Client) PutResourceWithDecorators(ctx context.Context, resourceID strin
 		return nil, clientErr
 	}
 
+	if !wait {
+		return resp, nil
+	}
 	response, err := c.WaitForAsyncOperationResult(ctx, future, "armclient.PutResource")
 	if err != nil {
 		if response != nil {
diff --git a/pkg/azureclients/armclient/interface.go b/pkg/azureclients/armclient/interface.go
index 4ce0ab085..6e38d6846 100644
--- a/pkg/azureclients/armclient/interface.go
+++ b/pkg/azureclients/armclient/interface.go
@@ -76,6 +76,9 @@ type Interface interface {
 	// PutResourceWithDecorators puts a resource with decorators by resource ID
 	PutResourceWithDecorators(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error)
 
+	// PutResourceWithDecoratorsAsync puts a resource with decorators by resource ID
+	PutResourceWithDecoratorsAsync(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error)
+
 	// PatchResource patches a resource by resource ID
 	PatchResource(ctx context.Context, resourceID string, parameters interface{}) (*http.Response, *retry.Error)
 
diff --git a/pkg/azureclients/armclient/mockarmclient/interface.go b/pkg/azureclients/armclient/mockarmclient/interface.go
index adcb549ba..0af9b7507 100644
--- a/pkg/azureclients/armclient/mockarmclient/interface.go
+++ b/pkg/azureclients/armclient/mockarmclient/interface.go
@@ -337,6 +337,21 @@ func (mr *MockInterfaceMockRecorder) PutResourceWithDecorators(ctx, resourceID,
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResourceWithDecorators", reflect.TypeOf((*MockInterface)(nil).PutResourceWithDecorators), ctx, resourceID, parameters, decorators)
 }
 
+// PutResourceWithDecoratorsAsync mocks base method.
+func (m *MockInterface) PutResourceWithDecoratorsAsync(ctx context.Context, resourceID string, parameters interface{}, decorators []autorest.PrepareDecorator) (*http.Response, *retry.Error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "PutResourceWithDecoratorsAsync", ctx, resourceID, parameters, decorators)
+	ret0, _ := ret[0].(*http.Response)
+	ret1, _ := ret[1].(*retry.Error)
+	return ret0, ret1
+}
+
+// PutResourceWithDecoratorsAsync indicates an expected call of PutResourceWithDecoratorsAsync.
+func (mr *MockInterfaceMockRecorder) PutResourceWithDecoratorsAsync(ctx, resourceID, parameters, decorators interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutResourceWithDecoratorsAsync", reflect.TypeOf((*MockInterface)(nil).PutResourceWithDecoratorsAsync), ctx, resourceID, parameters, decorators)
+}
+
 // PutResources mocks base method.
 func (m *MockInterface) PutResources(ctx context.Context, resources map[string]interface{}) map[string]*armclient.PutResourcesResponse {
 	m.ctrl.T.Helper()
diff --git a/pkg/azureclients/containerserviceclient/azure_containerserviceclient.go b/pkg/azureclients/containerserviceclient/azure_containerserviceclient.go
index 7555494c8..31d095469 100644
--- a/pkg/azureclients/containerserviceclient/azure_containerserviceclient.go
+++ b/pkg/azureclients/containerserviceclient/azure_containerserviceclient.go
@@ -37,6 +37,8 @@ import (
 
 var _ Interface = &Client{}
 
+const managedClusterResourceType = "Microsoft.ContainerService/managedClusters"
+
 // Client implements ContainerService client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -120,7 +122,7 @@ func (c *Client) getManagedCluster(ctx context.Context, resourceGroupName string
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.ContainerService/managedClusters",
+		managedClusterResourceType,
 		managedClusterName,
 	)
 	result := containerservice.ManagedCluster{}
@@ -178,7 +180,7 @@ func (c *Client) List(ctx context.Context, resourceGroupName string) ([]containe
 
 // listManagedCluster gets a list of ManagedClusters in the resource group.
 func (c *Client) listManagedCluster(ctx context.Context, resourceGroupName string) ([]containerservice.ManagedCluster, *retry.Error) {
-	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, "Microsoft.ContainerService/managedClusters")
+	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, managedClusterResourceType)
 	result := make([]containerservice.ManagedCluster, 0)
 	page := &ManagedClusterResultPage{}
 	page.fn = c.listNextResults
@@ -340,7 +342,7 @@ func (c *Client) createOrUpdateManagedCluster(ctx context.Context, resourceGroup
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.ContainerService/managedClusters",
+		managedClusterResourceType,
 		managedClusterName,
 	)
 	decorators := []autorest.PrepareDecorator{
@@ -415,7 +417,7 @@ func (c *Client) deleteManagedCluster(ctx context.Context, resourceGroupName str
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.ContainerService/managedClusters",
+		managedClusterResourceType,
 		managedClusterName,
 	)
 
diff --git a/pkg/azureclients/deploymentclient/azure_deploymentclient.go b/pkg/azureclients/deploymentclient/azure_deploymentclient.go
index 18d8819cb..737b83722 100644
--- a/pkg/azureclients/deploymentclient/azure_deploymentclient.go
+++ b/pkg/azureclients/deploymentclient/azure_deploymentclient.go
@@ -37,6 +37,8 @@ import (
 
 var _ Interface = &Client{}
 
+const deploymentResourceType = "Microsoft.Resources/deployments"
+
 // Client implements ContainerService client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -113,7 +115,7 @@ func (c *Client) getDeployment(ctx context.Context, resourceGroupName string, de
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Resources/deployments",
+		deploymentResourceType,
 		deploymentName,
 	)
 	result := resources.DeploymentExtended{}
@@ -171,7 +173,7 @@ func (c *Client) List(ctx context.Context, resourceGroupName string) ([]resource
 
 // listDeployment gets a list of deployments in the resource group.
 func (c *Client) listDeployment(ctx context.Context, resourceGroupName string) ([]resources.DeploymentExtended, *retry.Error) {
-	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, "Microsoft.Resources/deployments")
+	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, deploymentResourceType)
 	result := make([]resources.DeploymentExtended, 0)
 	page := &DeploymentResultPage{}
 	page.fn = c.listNextResults
@@ -332,7 +334,7 @@ func (c *Client) createOrUpdateDeployment(ctx context.Context, resourceGroupName
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Resources/deployments",
+		deploymentResourceType,
 		deploymentName,
 	)
 	decorators := []autorest.PrepareDecorator{
@@ -407,7 +409,7 @@ func (c *Client) deleteDeployment(ctx context.Context, resourceGroupName string,
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Resources/deployments",
+		deploymentResourceType,
 		deploymentName,
 	)
 
@@ -434,7 +436,7 @@ func (c *Client) ExportTemplate(ctx context.Context, resourceGroupName string, d
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Resources/deployments",
+		deploymentResourceType,
 		deploymentName,
 	)
 	response, rerr := c.armClient.PostResource(ctx, resourceID, "exportTemplate", struct{}{}, map[string]interface{}{})
diff --git a/pkg/azureclients/diskclient/azure_diskclient.go b/pkg/azureclients/diskclient/azure_diskclient.go
index a468d0dc8..0e4c57045 100644
--- a/pkg/azureclients/diskclient/azure_diskclient.go
+++ b/pkg/azureclients/diskclient/azure_diskclient.go
@@ -38,6 +38,8 @@ import (
 
 var _ Interface = &Client{}
 
+const diskResourceType = "Microsoft.Compute/disks"
+
 // Client implements Disk client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -126,7 +128,7 @@ func (c *Client) getDisk(ctx context.Context, subsID, resourceGroupName, diskNam
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Compute/disks",
+		diskResourceType,
 		diskName,
 	)
 	result := compute.Disk{}
@@ -190,7 +192,7 @@ func (c *Client) createOrUpdateDisk(ctx context.Context, subsID, resourceGroupNa
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Compute/disks",
+		diskResourceType,
 		diskName,
 	)
 
@@ -261,7 +263,7 @@ func (c *Client) updateDisk(ctx context.Context, subsID, resourceGroupName, disk
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Compute/disks",
+		diskResourceType,
 		diskName,
 	)
 
@@ -332,7 +334,7 @@ func (c *Client) deleteDisk(ctx context.Context, subsID, resourceGroupName strin
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Compute/disks",
+		diskResourceType,
 		diskName,
 	)
 
@@ -344,7 +346,7 @@ func (c *Client) ListByResourceGroup(ctx context.Context, subsID, resourceGroupN
 	if subsID == "" {
 		subsID = c.subscriptionID
 	}
-	resourceID := armclient.GetResourceListID(subsID, resourceGroupName, "Microsoft.Compute/disks")
+	resourceID := armclient.GetResourceListID(subsID, resourceGroupName, diskResourceType)
 
 	result := make([]compute.Disk, 0)
 	page := &DiskListPage{}
diff --git a/pkg/azureclients/interfaceclient/azure_interfaceclient.go b/pkg/azureclients/interfaceclient/azure_interfaceclient.go
index 4909e88ea..8faa5b734 100644
--- a/pkg/azureclients/interfaceclient/azure_interfaceclient.go
+++ b/pkg/azureclients/interfaceclient/azure_interfaceclient.go
@@ -39,6 +39,8 @@ import (
 
 var _ Interface = &Client{}
 
+const netInterfaceResourceType = "Microsoft.Network/networkInterfaces"
+
 // Client implements network interface client.
 type Client struct {
 	armClient              armclient.Interface
@@ -123,7 +125,7 @@ func (c *Client) getNetworkInterface(ctx context.Context, resourceGroupName stri
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/networkInterfaces",
+		netInterfaceResourceType,
 		networkInterfaceName,
 	)
 	result := network.Interface{}
@@ -259,7 +261,7 @@ func (c *Client) createOrUpdateInterface(ctx context.Context, resourceGroupName
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/networkInterfaces",
+		netInterfaceResourceType,
 		networkInterfaceName,
 	)
 	decorators := []autorest.PrepareDecorator{
@@ -334,7 +336,7 @@ func (c *Client) deleteInterface(ctx context.Context, resourceGroupName string,
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/networkInterfaces",
+		netInterfaceResourceType,
 		networkInterfaceName,
 	)
 
diff --git a/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go b/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go
index e8ce8ba78..51cb554f9 100644
--- a/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go
+++ b/pkg/azureclients/loadbalancerclient/azure_loadbalancerclient.go
@@ -38,6 +38,8 @@ import (
 
 var _ Interface = &Client{}
 
+const lbResourceType = "Microsoft.Network/loadBalancers"
+
 // Client implements LoadBalancer client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -120,7 +122,7 @@ func (c *Client) getLB(ctx context.Context, resourceGroupName string, loadBalanc
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/loadBalancers",
+		lbResourceType,
 		loadBalancerName,
 	)
 	result := network.LoadBalancer{}
@@ -250,7 +252,7 @@ func (c *Client) createOrUpdateLB(ctx context.Context, resourceGroupName string,
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/loadBalancers",
+		lbResourceType,
 		loadBalancerName,
 	)
 	decorators := []autorest.PrepareDecorator{
@@ -325,7 +327,7 @@ func (c *Client) deleteLB(ctx context.Context, resourceGroupName string, loadBal
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/loadBalancers",
+		lbResourceType,
 		loadBalancerName,
 	)
 
@@ -458,7 +460,7 @@ func (c *Client) createOrUpdateLBBackendPool(ctx context.Context, resourceGroupN
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/loadBalancers",
+		lbResourceType,
 		loadBalancerName,
 		"backendAddressPools",
 		backendPoolName,
@@ -524,7 +526,7 @@ func (c *Client) deleteLBBackendPool(ctx context.Context, resourceGroupName, loa
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/loadBalancers",
+		lbResourceType,
 		loadBalancerName,
 		"backendAddressPools",
 		backendPoolName,
diff --git a/pkg/azureclients/privatednsclient/azure_privatednsclient.go b/pkg/azureclients/privatednsclient/azure_privatednsclient.go
index 0c2369b9b..8f4a6290e 100644
--- a/pkg/azureclients/privatednsclient/azure_privatednsclient.go
+++ b/pkg/azureclients/privatednsclient/azure_privatednsclient.go
@@ -18,48 +18,216 @@ package privatednsclient
 
 import (
 	"context"
+	"net/http"
+	"strings"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure"
+
+	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/klog/v2"
+
 	azclients "sigs.k8s.io/cloud-provider-azure/pkg/azureclients"
+	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient"
+	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
+	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
 var _ Interface = &Client{}
 
+const (
+	privateDNSZoneResourceType = "Microsoft.Network/privateDnsZones"
+)
+
 // Client implements privatednsclient Interface.
 type Client struct {
-	privateDNSClient privatedns.PrivateZonesClient
+	armClient      armclient.Interface
+	subscriptionID string
+	cloudName      string
+
+	// Rate limiting configures.
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
+
+	// ARM throttling configures.
+	RetryAfterReader time.Time
+	RetryAfterWriter time.Time
 }
 
-// New creates a new privatedns client.
+// New creates a new private dns client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
-	privateDNSClient := privatedns.NewPrivateZonesClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionID)
-	privateDNSClient.Authorizer = config.Authorizer
+	baseURI := config.ResourceManagerEndpoint
+	authorizer := config.Authorizer
+	apiVersion := APIVersion
+	if strings.EqualFold(config.CloudName, AzureStackCloudName) && !config.DisableAzureStackCloud {
+		klog.Warningf("Azure Stack is not supported for Private DNS Zone API")
+	}
+	armClient := armclient.New(authorizer, *config, baseURI, apiVersion)
+	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
+
+	if azclients.RateLimitEnabled(config.RateLimitConfig) {
+		klog.V(2).Infof("Azure PrivateDNSZoneClient (read ops) using rate limit config: QPS=%g, bucket=%d",
+			config.RateLimitConfig.CloudProviderRateLimitQPS,
+			config.RateLimitConfig.CloudProviderRateLimitBucket)
+		klog.V(2).Infof("Azure PrivateDNSZoneClient (write ops) using rate limit config: QPS=%g, bucket=%d",
+			config.RateLimitConfig.CloudProviderRateLimitQPSWrite,
+			config.RateLimitConfig.CloudProviderRateLimitBucketWrite)
+	}
+
 	client := &Client{
-		privateDNSClient: privateDNSClient,
+		armClient:         armClient,
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+		subscriptionID:    config.SubscriptionID,
+		cloudName:         config.CloudName,
 	}
+
 	return client
 }
 
-// CreateOrUpdate creates or updates a private dns zone
-func (c *Client) CreateOrUpdate(ctx context.Context, resourceGroupName string, privateZoneName string, parameters privatedns.PrivateZone, waitForCompletion bool) error {
-	createOrUpdateFuture, err := c.privateDNSClient.CreateOrUpdate(ctx, resourceGroupName, privateZoneName, parameters, "", "*")
+// CreateOrUpdate creates or updates a private DNS zone.
+func (c *Client) CreateOrUpdate(ctx context.Context, resourceGroupName, privateZoneName string, parameters privatedns.PrivateZone, etag string, waitForCompletion bool) *retry.Error {
+	mc := metrics.NewMetricContext("private_dns_zone", "create_or_update", resourceGroupName, c.subscriptionID, "")
 
-	if err != nil {
-		klog.V(5).Infof("Received error for %s, resourceGroup: %s, error: %s", "privatedns.put.request", resourceGroupName, err)
-		return err
+	// Report errors if the client is rate limited.
+	if !c.rateLimiterWriter.TryAccept() {
+		mc.RateLimitedCount()
+		return retry.GetRateLimitError(true, "PrivateDNSZoneCreateOrUpdate")
+	}
+
+	// Report errors if the client is throttled.
+	if c.RetryAfterWriter.After(time.Now()) {
+		mc.ThrottledCount()
+		rerr := retry.GetThrottlingError("PrivateDNSZoneCreateOrUpdate", "client throttled", c.RetryAfterWriter)
+		return rerr
+	}
+
+	rerr := c.createOrUpdatePrivateDNSZone(ctx, resourceGroupName, privateZoneName, parameters, etag, waitForCompletion)
+	mc.Observe(rerr)
+	if rerr != nil {
+		if rerr.IsThrottled() {
+			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
+			c.RetryAfterWriter = rerr.RetryAfter
+		}
+
+		return rerr
+	}
+
+	return nil
+}
+
+// createOrUpdatePrivateDNSZone creates or updates a private DNS zone.
+func (c *Client) createOrUpdatePrivateDNSZone(ctx context.Context, resourceGroupName, privateDNSZoneName string, parameters privatedns.PrivateZone, etag string, waitForCompletion bool) *retry.Error {
+	resourceID := armclient.GetResourceID(
+		c.subscriptionID,
+		resourceGroupName,
+		privateDNSZoneResourceType,
+		privateDNSZoneName,
+	)
+	decorators := []autorest.PrepareDecorator{
+		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
+		autorest.WithJSON(parameters),
+	}
+	if etag != "" {
+		decorators = append(decorators, autorest.WithHeader("If-Match", autorest.String(etag)))
 	}
 
+	var response *http.Response
+	var rerr *retry.Error
 	if waitForCompletion {
-		err := createOrUpdateFuture.WaitForCompletionRef(ctx, c.privateDNSClient.Client)
-		if err != nil {
-			klog.V(5).Infof("Received error while waiting for completion for %s, resourceGroup: %s, error: %s", "privatedns.put.request", resourceGroupName, err)
-			return err
+		response, rerr = c.armClient.PutResourceWithDecorators(ctx, resourceID, parameters, decorators)
+	} else {
+		response, rerr = c.armClient.PutResourceWithDecoratorsAsync(ctx, resourceID, parameters, decorators)
+	}
+	defer c.armClient.CloseResponse(ctx, response)
+	if rerr != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "privatednszone.put.request", resourceID, rerr.Error())
+		return rerr
+	}
+	if !waitForCompletion {
+		return nil
+	}
+
+	if response != nil && response.StatusCode != http.StatusNoContent {
+		_, rerr = c.createOrUpdateResponder(response)
+		if rerr != nil {
+			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "privatednszone.put.respond", resourceID, rerr.Error())
+			return rerr
 		}
 	}
+
 	return nil
 }
 
-func (c *Client) Get(ctx context.Context, resourceGroupName string, privateZoneName string) (result privatedns.PrivateZone, err error) {
-	return c.privateDNSClient.Get(ctx, resourceGroupName, privateZoneName)
+func (c *Client) createOrUpdateResponder(resp *http.Response) (*privatedns.PrivateZone, *retry.Error) {
+	result := &privatedns.PrivateZone{}
+	err := autorest.Respond(
+		resp,
+		azure.WithErrorUnlessStatusCode(http.StatusOK, http.StatusCreated),
+		autorest.ByUnmarshallingJSON(&result))
+	result.Response = autorest.Response{Response: resp}
+	return result, retry.GetError(resp, err)
+}
+
+// Get gets a private dns zone.
+func (c *Client) Get(ctx context.Context, resourceGroupName, privateZoneName string) (privatedns.PrivateZone, *retry.Error) {
+	mc := metrics.NewMetricContext("private_dns_zones", "get", resourceGroupName, c.subscriptionID, "")
+
+	// Report errors if the client is rate limited.
+	if !c.rateLimiterReader.TryAccept() {
+		mc.RateLimitedCount()
+		return privatedns.PrivateZone{}, retry.GetRateLimitError(false, "PrivateDNSZoneGet")
+	}
+
+	// Report errors if the client is throttled.
+	if c.RetryAfterReader.After(time.Now()) {
+		mc.ThrottledCount()
+		rerr := retry.GetThrottlingError("PrivateDNSZoneGet", "client throttled", c.RetryAfterReader)
+		return privatedns.PrivateZone{}, rerr
+	}
+
+	result, rerr := c.getPrivateDNSZone(ctx, resourceGroupName, privateZoneName)
+	mc.Observe(rerr)
+	if rerr != nil {
+		if rerr.IsThrottled() {
+			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
+			c.RetryAfterReader = rerr.RetryAfter
+		}
+
+		return result, rerr
+	}
+
+	return result, nil
+}
+
+// getPrivateDNSZone gets a private DNS zone.
+func (c *Client) getPrivateDNSZone(ctx context.Context, resourceGroupName, privateDNSZoneName string) (privatedns.PrivateZone, *retry.Error) {
+	resourceID := armclient.GetResourceID(
+		c.subscriptionID,
+		resourceGroupName,
+		privateDNSZoneResourceType,
+		privateDNSZoneName,
+	)
+	result := privatedns.PrivateZone{}
+
+	response, rerr := c.armClient.GetResource(ctx, resourceID)
+	defer c.armClient.CloseResponse(ctx, response)
+	if rerr != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "privatednszone.get.request", resourceID, rerr.Error())
+		return result, rerr
+	}
+
+	err := autorest.Respond(
+		response,
+		azure.WithErrorUnlessStatusCode(http.StatusOK),
+		autorest.ByUnmarshallingJSON(&result))
+	if err != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "privatednszone.get.respond", resourceID, err)
+		return result, retry.GetError(response, err)
+	}
+
+	result.Response = autorest.Response{Response: response}
+	return result, nil
 }
diff --git a/pkg/azureclients/privatednsclient/interface.go b/pkg/azureclients/privatednsclient/interface.go
index 628f8dbc4..63ae62a0a 100644
--- a/pkg/azureclients/privatednsclient/interface.go
+++ b/pkg/azureclients/privatednsclient/interface.go
@@ -20,15 +20,23 @@ import (
 	"context"
 
 	"github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
+
+	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
+)
+
+const (
+	// APIVersion is the API version.
+	APIVersion = "2018-09-01"
+	// AzureStackCloudName is the cloud name of Azure Stack
+	AzureStackCloudName = "AZURESTACKCLOUD"
 )
 
 // Interface is the client interface for Private DNS Zones
 // Don't forget to run "hack/update-mock-clients.sh" command to generate the mock client.
 type Interface interface {
+	// Get gets a private DNS zone
+	Get(ctx context.Context, resourceGroupName, privateZoneName string) (privatedns.PrivateZone, *retry.Error)
 
-	//Get gets the PrivateDNSZone
-	Get(ctx context.Context, resourceGroupName string, privateZoneName string) (result privatedns.PrivateZone, err error)
-
-	// CreateOrUpdate creates or updates a private dns zone.
-	CreateOrUpdate(ctx context.Context, resourceGroupName string, privateZoneName string, parameters privatedns.PrivateZone, waitForCompletion bool) error
+	// CreateOrUpdate creates or updates a private DNS zone.
+	CreateOrUpdate(ctx context.Context, resourceGroupName, privateZoneName string, parameters privatedns.PrivateZone, etag string, waitForCompletion bool) *retry.Error
 }
diff --git a/pkg/azureclients/privatednsclient/mockprivatednsclient/interface.go b/pkg/azureclients/privatednsclient/mockprivatednsclient/interface.go
index 5999d2356..0bd0dd21d 100644
--- a/pkg/azureclients/privatednsclient/mockprivatednsclient/interface.go
+++ b/pkg/azureclients/privatednsclient/mockprivatednsclient/interface.go
@@ -27,6 +27,7 @@ import (
 
 	privatedns "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
 	gomock "github.com/golang/mock/gomock"
+	retry "sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
 // MockInterface is a mock of Interface interface.
@@ -53,25 +54,25 @@ func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 }
 
 // CreateOrUpdate mocks base method.
-func (m *MockInterface) CreateOrUpdate(ctx context.Context, resourceGroupName, privateZoneName string, parameters privatedns.PrivateZone, waitForCompletion bool) error {
+func (m *MockInterface) CreateOrUpdate(ctx context.Context, resourceGroupName, privateZoneName string, parameters privatedns.PrivateZone, etag string, waitForCompletion bool) *retry.Error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "CreateOrUpdate", ctx, resourceGroupName, privateZoneName, parameters, waitForCompletion)
-	ret0, _ := ret[0].(error)
+	ret := m.ctrl.Call(m, "CreateOrUpdate", ctx, resourceGroupName, privateZoneName, parameters, etag, waitForCompletion)
+	ret0, _ := ret[0].(*retry.Error)
 	return ret0
 }
 
 // CreateOrUpdate indicates an expected call of CreateOrUpdate.
-func (mr *MockInterfaceMockRecorder) CreateOrUpdate(ctx, resourceGroupName, privateZoneName, parameters, waitForCompletion interface{}) *gomock.Call {
+func (mr *MockInterfaceMockRecorder) CreateOrUpdate(ctx, resourceGroupName, privateZoneName, parameters, etag, waitForCompletion interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockInterface)(nil).CreateOrUpdate), ctx, resourceGroupName, privateZoneName, parameters, waitForCompletion)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockInterface)(nil).CreateOrUpdate), ctx, resourceGroupName, privateZoneName, parameters, etag, waitForCompletion)
 }
 
 // Get mocks base method.
-func (m *MockInterface) Get(ctx context.Context, resourceGroupName, privateZoneName string) (privatedns.PrivateZone, error) {
+func (m *MockInterface) Get(ctx context.Context, resourceGroupName, privateZoneName string) (privatedns.PrivateZone, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, privateZoneName)
 	ret0, _ := ret[0].(privatedns.PrivateZone)
-	ret1, _ := ret[1].(error)
+	ret1, _ := ret[1].(*retry.Error)
 	return ret0, ret1
 }
 
diff --git a/pkg/azureclients/privatednszonegroupclient/azure_privatednszonegroupclient.go b/pkg/azureclients/privatednszonegroupclient/azure_privatednszonegroupclient.go
index 401d7d0d3..79719efa8 100644
--- a/pkg/azureclients/privatednszonegroupclient/azure_privatednszonegroupclient.go
+++ b/pkg/azureclients/privatednszonegroupclient/azure_privatednszonegroupclient.go
@@ -18,47 +18,220 @@ package privatednszonegroupclient
 
 import (
 	"context"
+	"net/http"
+	"strings"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure"
+
+	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/klog/v2"
+
 	azclients "sigs.k8s.io/cloud-provider-azure/pkg/azureclients"
+	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient"
+	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
+	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
 var _ Interface = &Client{}
 
+const (
+	peResourceType                  = "Microsoft.Network/privateEndpoints"
+	privateDNSZoneGroupResourceType = "privateDnsZoneGroups"
+)
+
 // Client implements privatednszonegroupclient client Interface.
 type Client struct {
-	privateDNSZoneGroupClient network.PrivateDNSZoneGroupsClient
+	armClient      armclient.Interface
+	subscriptionID string
+	cloudName      string
+
+	// Rate limiting configures.
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
+
+	// ARM throttling configures.
+	RetryAfterReader time.Time
+	RetryAfterWriter time.Time
 }
 
-// New creates a new private dns zone group client.
+// New creates a new private dns zone group client with ratelimiting.
 func New(config *azclients.ClientConfig) *Client {
-	privateDNSZoneGroupClient := network.NewPrivateDNSZoneGroupsClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionID)
-	privateDNSZoneGroupClient.Authorizer = config.Authorizer
+	baseURI := config.ResourceManagerEndpoint
+	authorizer := config.Authorizer
+	apiVersion := APIVersion
+	if strings.EqualFold(config.CloudName, AzureStackCloudName) && !config.DisableAzureStackCloud {
+		klog.Warningf("Azure Stack is not supported for Private DNS Zone Group API")
+	}
+	armClient := armclient.New(authorizer, *config, baseURI, apiVersion)
+	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(config.RateLimitConfig)
+
+	if azclients.RateLimitEnabled(config.RateLimitConfig) {
+		klog.V(2).Infof("Azure PrivateDNSZoneGroupClient (read ops) using rate limit config: QPS=%g, bucket=%d",
+			config.RateLimitConfig.CloudProviderRateLimitQPS,
+			config.RateLimitConfig.CloudProviderRateLimitBucket)
+		klog.V(2).Infof("Azure PrivateDNSZoneGroupClient (write ops) using rate limit config: QPS=%g, bucket=%d",
+			config.RateLimitConfig.CloudProviderRateLimitQPSWrite,
+			config.RateLimitConfig.CloudProviderRateLimitBucketWrite)
+	}
+
 	client := &Client{
-		privateDNSZoneGroupClient: privateDNSZoneGroupClient,
+		armClient:         armClient,
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+		subscriptionID:    config.SubscriptionID,
+		cloudName:         config.CloudName,
 	}
+
 	return client
 }
 
-// CreateOrUpdate creates or updates a private dns zone group
-func (c *Client) CreateOrUpdate(ctx context.Context, resourceGroupName string, privateEndpointName string, privateDNSZoneGroupName string, parameters network.PrivateDNSZoneGroup, waitForCompletion bool) error {
-	createOrUpdateFuture, err := c.privateDNSZoneGroupClient.CreateOrUpdate(ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName, parameters)
-	if err != nil {
-		klog.V(5).Infof("Received error for %s, resourceGroup: %s, privateEndpointName: %s, error: %s", "privatednszonegroup.put.request", resourceGroupName, privateEndpointName, err)
-		return err
+// CreateOrUpdate creates or updates a private DNS zone group.
+func (c *Client) CreateOrUpdate(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string, parameters network.PrivateDNSZoneGroup, etag string, waitForCompletion bool) *retry.Error {
+	mc := metrics.NewMetricContext("private_dns_zone_group", "create_or_update", resourceGroupName, c.subscriptionID, "")
+
+	// Report errors if the client is rate limited.
+	if !c.rateLimiterWriter.TryAccept() {
+		mc.RateLimitedCount()
+		return retry.GetRateLimitError(true, "PrivateDNSZoneGroupCreateOrUpdate")
+	}
+
+	// Report errors if the client is throttled.
+	if c.RetryAfterWriter.After(time.Now()) {
+		mc.ThrottledCount()
+		rerr := retry.GetThrottlingError("PrivateDNSZoneGroupCreateOrUpdate", "client throttled", c.RetryAfterWriter)
+		return rerr
+	}
+
+	rerr := c.createOrUpdatePrivateDNSZoneGroup(ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName, parameters, etag, waitForCompletion)
+	mc.Observe(rerr)
+	if rerr != nil {
+		if rerr.IsThrottled() {
+			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
+			c.RetryAfterWriter = rerr.RetryAfter
+		}
+
+		return rerr
+	}
+
+	return nil
+}
+
+// createOrUpdatePrivateDNSZoneGroup creates or updates a private DNS zone group.
+func (c *Client) createOrUpdatePrivateDNSZoneGroup(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string, parameters network.PrivateDNSZoneGroup, etag string, waitForCompletion bool) *retry.Error {
+	resourceID := armclient.GetChildResourceID(
+		c.subscriptionID,
+		resourceGroupName,
+		peResourceType,
+		privateEndpointName,
+		privateDNSZoneGroupResourceType,
+		privateDNSZoneGroupName)
+	decorators := []autorest.PrepareDecorator{
+		autorest.WithPathParameters("{resourceID}", map[string]interface{}{"resourceID": resourceID}),
+		autorest.WithJSON(parameters),
+	}
+	if etag != "" {
+		decorators = append(decorators, autorest.WithHeader("If-Match", autorest.String(etag)))
 	}
+
+	var response *http.Response
+	var rerr *retry.Error
 	if waitForCompletion {
-		err = createOrUpdateFuture.WaitForCompletionRef(ctx, c.privateDNSZoneGroupClient.Client)
-		if err != nil {
-			klog.V(5).Infof("Received error while waiting for completion for %s, resourceGroup: %s, privateEndpointName: %s, error: %s", "privatednszonegroup.put.request", resourceGroupName, privateEndpointName, err)
-			return err
+		response, rerr = c.armClient.PutResourceWithDecorators(ctx, resourceID, parameters, decorators)
+	} else {
+		response, rerr = c.armClient.PutResourceWithDecoratorsAsync(ctx, resourceID, parameters, decorators)
+	}
+	defer c.armClient.CloseResponse(ctx, response)
+	if rerr != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "privatednszonegroup.put.request", resourceID, rerr.Error())
+		return rerr
+	}
+	if !waitForCompletion {
+		return nil
+	}
+
+	if response != nil && response.StatusCode != http.StatusNoContent {
+		_, rerr = c.createOrUpdateResponder(response)
+		if rerr != nil {
+			klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "privatednszonegroup.put.respond", resourceID, rerr.Error())
+			return rerr
 		}
 	}
+
 	return nil
 }
 
-// Get gets the private dns zone group
-func (c *Client) Get(ctx context.Context, resourceGroupName string, privateEndpointName string, privateDNSZoneGroupName string) (result network.PrivateDNSZoneGroup, err error) {
-	return c.privateDNSZoneGroupClient.Get(ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName)
+func (c *Client) createOrUpdateResponder(resp *http.Response) (*network.PrivateDNSZoneGroup, *retry.Error) {
+	result := &network.PrivateDNSZoneGroup{}
+	err := autorest.Respond(
+		resp,
+		azure.WithErrorUnlessStatusCode(http.StatusOK, http.StatusCreated),
+		autorest.ByUnmarshallingJSON(&result))
+	result.Response = autorest.Response{Response: resp}
+	return result, retry.GetError(resp, err)
+}
+
+// Get gets a private dns zone group.
+func (c *Client) Get(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string) (network.PrivateDNSZoneGroup, *retry.Error) {
+	mc := metrics.NewMetricContext("private_dns_zone_group", "get", resourceGroupName, c.subscriptionID, "")
+
+	// Report errors if the client is rate limited.
+	if !c.rateLimiterReader.TryAccept() {
+		mc.RateLimitedCount()
+		return network.PrivateDNSZoneGroup{}, retry.GetRateLimitError(false, "PrivateDNSZoneGroupGet")
+	}
+
+	// Report errors if the client is throttled.
+	if c.RetryAfterReader.After(time.Now()) {
+		mc.ThrottledCount()
+		rerr := retry.GetThrottlingError("PrivateDNSZoneGroupGet", "client throttled", c.RetryAfterReader)
+		return network.PrivateDNSZoneGroup{}, rerr
+	}
+
+	result, rerr := c.getPrivateDNSZoneGroup(ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName)
+	mc.Observe(rerr)
+	if rerr != nil {
+		if rerr.IsThrottled() {
+			// Update RetryAfterReader so that no more requests would be sent until RetryAfter expires.
+			c.RetryAfterReader = rerr.RetryAfter
+		}
+
+		return result, rerr
+	}
+
+	return result, nil
+}
+
+// getPrivateDNSZoneGroup gets a private DNS zone group.
+func (c *Client) getPrivateDNSZoneGroup(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string) (network.PrivateDNSZoneGroup, *retry.Error) {
+	resourceID := armclient.GetChildResourceID(
+		c.subscriptionID,
+		resourceGroupName,
+		peResourceType,
+		privateEndpointName,
+		privateDNSZoneGroupResourceType,
+		privateDNSZoneGroupName,
+	)
+	result := network.PrivateDNSZoneGroup{}
+
+	response, rerr := c.armClient.GetResource(ctx, resourceID)
+	defer c.armClient.CloseResponse(ctx, response)
+	if rerr != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "privatednszonegroup.get.request", resourceID, rerr.Error())
+		return result, rerr
+	}
+
+	err := autorest.Respond(
+		response,
+		azure.WithErrorUnlessStatusCode(http.StatusOK),
+		autorest.ByUnmarshallingJSON(&result))
+	if err != nil {
+		klog.V(5).Infof("Received error in %s: resourceID: %s, error: %s", "privatednszonegroup.get.respond", resourceID, err)
+		return result, retry.GetError(response, err)
+	}
+
+	result.Response = autorest.Response{Response: response}
+	return result, nil
 }
diff --git a/pkg/azureclients/privatednszonegroupclient/interface.go b/pkg/azureclients/privatednszonegroupclient/interface.go
index fae7df643..72c531b18 100644
--- a/pkg/azureclients/privatednszonegroupclient/interface.go
+++ b/pkg/azureclients/privatednszonegroupclient/interface.go
@@ -20,15 +20,23 @@ import (
 	"context"
 
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network"
+
+	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
+)
+
+const (
+	// APIVersion is the API version.
+	APIVersion = "2021-02-01"
+	// AzureStackCloudName is the cloud name of Azure Stack
+	AzureStackCloudName = "AZURESTACKCLOUD"
 )
 
 // Interface is the client interface for Private DNS Zone Group.
 // Don't forget to run "hack/update-mock-clients.sh" command to generate the mock client.
 type Interface interface {
-
 	// Get gets the private dns zone group
-	Get(ctx context.Context, resourceGroupName string, privateEndpointName string, privateDNSZoneGroupName string) (result network.PrivateDNSZoneGroup, err error)
+	Get(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string) (network.PrivateDNSZoneGroup, *retry.Error)
 
-	// CreateOrUpdate creates or updates a private dns zone group endpoint.
-	CreateOrUpdate(ctx context.Context, resourceGroupName string, privateEndpointName string, privateDNSZoneGroupName string, parameters network.PrivateDNSZoneGroup, waitForCompletion bool) error
+	// CreateOrUpdate creates or updates a private dns zone group.
+	CreateOrUpdate(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string, parameters network.PrivateDNSZoneGroup, etag string, waitForCompletion bool) *retry.Error
 }
diff --git a/pkg/azureclients/privatednszonegroupclient/mockprivatednszonegroupclient/interface.go b/pkg/azureclients/privatednszonegroupclient/mockprivatednszonegroupclient/interface.go
index f752205a2..bd6fbc10d 100644
--- a/pkg/azureclients/privatednszonegroupclient/mockprivatednszonegroupclient/interface.go
+++ b/pkg/azureclients/privatednszonegroupclient/mockprivatednszonegroupclient/interface.go
@@ -27,6 +27,7 @@ import (
 
 	network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network"
 	gomock "github.com/golang/mock/gomock"
+	retry "sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
 // MockInterface is a mock of Interface interface.
@@ -53,25 +54,25 @@ func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder {
 }
 
 // CreateOrUpdate mocks base method.
-func (m *MockInterface) CreateOrUpdate(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string, parameters network.PrivateDNSZoneGroup, waitForCompletion bool) error {
+func (m *MockInterface) CreateOrUpdate(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string, parameters network.PrivateDNSZoneGroup, etag string, waitForCompletion bool) *retry.Error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "CreateOrUpdate", ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName, parameters, waitForCompletion)
-	ret0, _ := ret[0].(error)
+	ret := m.ctrl.Call(m, "CreateOrUpdate", ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName, parameters, etag, waitForCompletion)
+	ret0, _ := ret[0].(*retry.Error)
 	return ret0
 }
 
 // CreateOrUpdate indicates an expected call of CreateOrUpdate.
-func (mr *MockInterfaceMockRecorder) CreateOrUpdate(ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName, parameters, waitForCompletion interface{}) *gomock.Call {
+func (mr *MockInterfaceMockRecorder) CreateOrUpdate(ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName, parameters, etag, waitForCompletion interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockInterface)(nil).CreateOrUpdate), ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName, parameters, waitForCompletion)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*MockInterface)(nil).CreateOrUpdate), ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName, parameters, etag, waitForCompletion)
 }
 
 // Get mocks base method.
-func (m *MockInterface) Get(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string) (network.PrivateDNSZoneGroup, error) {
+func (m *MockInterface) Get(ctx context.Context, resourceGroupName, privateEndpointName, privateDNSZoneGroupName string) (network.PrivateDNSZoneGroup, *retry.Error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Get", ctx, resourceGroupName, privateEndpointName, privateDNSZoneGroupName)
 	ret0, _ := ret[0].(network.PrivateDNSZoneGroup)
-	ret1, _ := ret[1].(error)
+	ret1, _ := ret[1].(*retry.Error)
 	return ret0, ret1
 }
 
diff --git a/pkg/azureclients/publicipclient/azure_publicipclient.go b/pkg/azureclients/publicipclient/azure_publicipclient.go
index 2b88ef07a..7f12b717b 100644
--- a/pkg/azureclients/publicipclient/azure_publicipclient.go
+++ b/pkg/azureclients/publicipclient/azure_publicipclient.go
@@ -39,6 +39,8 @@ import (
 
 var _ Interface = &Client{}
 
+const publicIPResourceType = "Microsoft.Network/publicIPAddresses"
+
 // Client implements PublicIPAddress client Interface.
 type Client struct {
 	armClient              armclient.Interface
@@ -123,7 +125,7 @@ func (c *Client) getPublicIPAddress(ctx context.Context, resourceGroupName strin
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/publicIPAddresses",
+		publicIPResourceType,
 		publicIPAddressName,
 	)
 	result := network.PublicIPAddress{}
@@ -258,7 +260,7 @@ func (c *Client) List(ctx context.Context, resourceGroupName string) ([]network.
 
 // listPublicIPAddress gets a list of PublicIPAddress in the resource group.
 func (c *Client) listPublicIPAddress(ctx context.Context, resourceGroupName string) ([]network.PublicIPAddress, *retry.Error) {
-	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, "Microsoft.Network/publicIPAddresses")
+	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, publicIPResourceType)
 	result := make([]network.PublicIPAddress, 0)
 	page := &PublicIPAddressListResultPage{}
 	page.fn = c.listNextResults
@@ -330,7 +332,7 @@ func (c *Client) createOrUpdatePublicIP(ctx context.Context, resourceGroupName s
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/publicIPAddresses",
+		publicIPResourceType,
 		publicIPAddressName,
 	)
 
@@ -398,7 +400,7 @@ func (c *Client) deletePublicIP(ctx context.Context, resourceGroupName string, p
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/publicIPAddresses",
+		publicIPResourceType,
 		publicIPAddressName,
 	)
 
@@ -523,7 +525,7 @@ func (c *Client) ListAll(ctx context.Context) ([]network.PublicIPAddress, *retry
 
 // listAllPublicIPAddress gets all of PublicIPAddress in the subscription.
 func (c *Client) listAllPublicIPAddress(ctx context.Context) ([]network.PublicIPAddress, *retry.Error) {
-	resourceID := armclient.GetProviderResourceID(c.subscriptionID, "Microsoft.Network/publicIPAddresses")
+	resourceID := armclient.GetProviderResourceID(c.subscriptionID, publicIPResourceType)
 	result := make([]network.PublicIPAddress, 0)
 	page := &PublicIPAddressListResultPage{}
 	page.fn = c.listNextResults
diff --git a/pkg/azureclients/routeclient/azure_routeclient.go b/pkg/azureclients/routeclient/azure_routeclient.go
index e70801760..2c4f65243 100644
--- a/pkg/azureclients/routeclient/azure_routeclient.go
+++ b/pkg/azureclients/routeclient/azure_routeclient.go
@@ -37,6 +37,8 @@ import (
 
 var _ Interface = &Client{}
 
+const routeTablesResourceType = "Microsoft.Network/routeTables"
+
 // Client implements Route client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -119,7 +121,7 @@ func (c *Client) createOrUpdateRoute(ctx context.Context, resourceGroupName stri
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/routeTables",
+		routeTablesResourceType,
 		routeTableName,
 		"routes",
 		routeName,
@@ -196,7 +198,7 @@ func (c *Client) deleteRoute(ctx context.Context, resourceGroupName string, rout
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/routeTables",
+		routeTablesResourceType,
 		routeTableName,
 		"routes",
 		routeName,
diff --git a/pkg/azureclients/routetableclient/azure_routetableclient.go b/pkg/azureclients/routetableclient/azure_routetableclient.go
index 2ba1fbb99..b66ac9ff9 100644
--- a/pkg/azureclients/routetableclient/azure_routetableclient.go
+++ b/pkg/azureclients/routetableclient/azure_routetableclient.go
@@ -37,6 +37,8 @@ import (
 
 var _ Interface = &Client{}
 
+const routeTablesResourceType = "Microsoft.Network/routeTables"
+
 // Client implements RouteTable client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -119,7 +121,7 @@ func (c *Client) getRouteTable(ctx context.Context, resourceGroupName string, ro
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/routeTables",
+		routeTablesResourceType,
 		routeTableName,
 	)
 	result := network.RouteTable{}
@@ -180,7 +182,7 @@ func (c *Client) createOrUpdateRouteTable(ctx context.Context, resourceGroupName
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/routeTables",
+		routeTablesResourceType,
 		routeTableName,
 	)
 	decorators := []autorest.PrepareDecorator{
diff --git a/pkg/azureclients/securitygroupclient/azure_securitygroupclient.go b/pkg/azureclients/securitygroupclient/azure_securitygroupclient.go
index eccbb78fc..9180a2b40 100644
--- a/pkg/azureclients/securitygroupclient/azure_securitygroupclient.go
+++ b/pkg/azureclients/securitygroupclient/azure_securitygroupclient.go
@@ -38,6 +38,8 @@ import (
 
 var _ Interface = &Client{}
 
+const nsgResourceType = "Microsoft.Network/networkSecurityGroups"
+
 // Client implements SecurityGroup client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -120,7 +122,7 @@ func (c *Client) getSecurityGroup(ctx context.Context, resourceGroupName string,
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/networkSecurityGroups",
+		nsgResourceType,
 		networkSecurityGroupName,
 	)
 	result := network.SecurityGroup{}
@@ -178,7 +180,7 @@ func (c *Client) List(ctx context.Context, resourceGroupName string) ([]network.
 
 // listSecurityGroup gets a list of SecurityGroups in the resource group.
 func (c *Client) listSecurityGroup(ctx context.Context, resourceGroupName string) ([]network.SecurityGroup, *retry.Error) {
-	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, "Microsoft.Network/networkSecurityGroups")
+	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, nsgResourceType)
 	result := make([]network.SecurityGroup, 0)
 	page := &SecurityGroupListResultPage{}
 	page.fn = c.listNextResults
@@ -250,7 +252,7 @@ func (c *Client) createOrUpdateNSG(ctx context.Context, resourceGroupName string
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/networkSecurityGroups",
+		nsgResourceType,
 		networkSecurityGroupName,
 	)
 	decorators := []autorest.PrepareDecorator{
@@ -325,7 +327,7 @@ func (c *Client) deleteNSG(ctx context.Context, resourceGroupName string, networ
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/networkSecurityGroups",
+		nsgResourceType,
 		networkSecurityGroupName,
 	)
 
diff --git a/pkg/azureclients/snapshotclient/azure_snapshotclient.go b/pkg/azureclients/snapshotclient/azure_snapshotclient.go
index 219b2caea..4e26705ea 100644
--- a/pkg/azureclients/snapshotclient/azure_snapshotclient.go
+++ b/pkg/azureclients/snapshotclient/azure_snapshotclient.go
@@ -38,6 +38,8 @@ import (
 
 var _ Interface = &Client{}
 
+const snapshotsResourceType = "Microsoft.Compute/snapshots"
+
 // Client implements Snapshot client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -123,7 +125,7 @@ func (c *Client) getSnapshot(ctx context.Context, subsID, resourceGroupName, sna
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Compute/snapshots",
+		snapshotsResourceType,
 		snapshotName,
 	)
 	result := compute.Snapshot{}
@@ -187,7 +189,7 @@ func (c *Client) deleteSnapshot(ctx context.Context, subsID, resourceGroupName,
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Compute/snapshots",
+		snapshotsResourceType,
 		snapshotName,
 	)
 
@@ -233,7 +235,7 @@ func (c *Client) createOrUpdateSnapshot(ctx context.Context, subsID, resourceGro
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Compute/snapshots",
+		snapshotsResourceType,
 		snapshotName,
 	)
 
@@ -304,7 +306,7 @@ func (c *Client) listSnapshotsByResourceGroup(ctx context.Context, subsID, resou
 	if subsID == "" {
 		subsID = c.subscriptionID
 	}
-	resourceID := armclient.GetResourceListID(subsID, resourceGroupName, "Microsoft.Compute/snapshots")
+	resourceID := armclient.GetResourceListID(subsID, resourceGroupName, snapshotsResourceType)
 	result := make([]compute.Snapshot, 0)
 	page := &SnapshotListPage{}
 	page.fn = c.listNextResults
diff --git a/pkg/azureclients/storageaccountclient/azure_storageaccountclient.go b/pkg/azureclients/storageaccountclient/azure_storageaccountclient.go
index 95d2bcdb6..7e73b4d55 100644
--- a/pkg/azureclients/storageaccountclient/azure_storageaccountclient.go
+++ b/pkg/azureclients/storageaccountclient/azure_storageaccountclient.go
@@ -38,6 +38,8 @@ import (
 
 var _ Interface = &Client{}
 
+const saResourceType = "Microsoft.Storage/storageAccounts"
+
 // Client implements StorageAccount client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -123,7 +125,7 @@ func (c *Client) getStorageAccount(ctx context.Context, subsID, resourceGroupNam
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Storage/storageAccounts",
+		saResourceType,
 		accountName,
 	)
 	result := storage.Account{}
@@ -187,7 +189,7 @@ func (c *Client) listStorageAccountKeys(ctx context.Context, subsID, resourceGro
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Storage/storageAccounts",
+		saResourceType,
 		accountName,
 	)
 
@@ -249,7 +251,7 @@ func (c *Client) createStorageAccount(ctx context.Context, subsID, resourceGroup
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Storage/storageAccounts",
+		saResourceType,
 		accountName,
 	)
 
@@ -318,7 +320,7 @@ func (c *Client) updateStorageAccount(ctx context.Context, subsID, resourceGroup
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Storage/storageAccounts",
+		saResourceType,
 		accountName,
 	)
 
@@ -389,7 +391,7 @@ func (c *Client) deleteStorageAccount(ctx context.Context, subsID, resourceGroup
 	resourceID := armclient.GetResourceID(
 		subsID,
 		resourceGroupName,
-		"Microsoft.Storage/storageAccounts",
+		saResourceType,
 		accountName,
 	)
 
@@ -433,7 +435,7 @@ func (c *Client) ListStorageAccountByResourceGroup(ctx context.Context, subsID,
 	if subsID == "" {
 		subsID = c.subscriptionID
 	}
-	resourceID := armclient.GetResourceListID(subsID, resourceGroupName, "Microsoft.Storage/storageAccounts")
+	resourceID := armclient.GetResourceListID(subsID, resourceGroupName, saResourceType)
 	result := make([]storage.Account, 0)
 	page := &AccountListResultPage{}
 	page.fn = c.listNextResults
diff --git a/pkg/azureclients/subnetclient/azure_subnetclient.go b/pkg/azureclients/subnetclient/azure_subnetclient.go
index 05ebfa0b4..e5d63401d 100644
--- a/pkg/azureclients/subnetclient/azure_subnetclient.go
+++ b/pkg/azureclients/subnetclient/azure_subnetclient.go
@@ -38,6 +38,8 @@ import (
 
 var _ Interface = &Client{}
 
+const vnetResourceType = "Microsoft.Network/virtualNetworks"
+
 // Client implements Subnet client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -120,7 +122,7 @@ func (c *Client) getSubnet(ctx context.Context, resourceGroupName string, virtua
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/virtualNetworks",
+		vnetResourceType,
 		virtualNetworkName,
 		"subnets",
 		subnetName,
@@ -183,7 +185,7 @@ func (c *Client) listSubnet(ctx context.Context, resourceGroupName string, virtu
 	resourceID := armclient.GetChildResourcesListID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/virtualNetworks",
+		vnetResourceType,
 		virtualNetworkName,
 		"subnets")
 
@@ -258,7 +260,7 @@ func (c *Client) createOrUpdateSubnet(ctx context.Context, resourceGroupName str
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/virtualNetworks",
+		vnetResourceType,
 		virtualNetworkName,
 		"subnets",
 		subnetName)
@@ -327,7 +329,7 @@ func (c *Client) deleteSubnet(ctx context.Context, resourceGroupName string, vir
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Network/virtualNetworks",
+		vnetResourceType,
 		virtualNetworkName,
 		"subnets",
 		subnetName)
diff --git a/pkg/azureclients/vmasclient/azure_vmasclient.go b/pkg/azureclients/vmasclient/azure_vmasclient.go
index 5169b0e6c..59372fb25 100644
--- a/pkg/azureclients/vmasclient/azure_vmasclient.go
+++ b/pkg/azureclients/vmasclient/azure_vmasclient.go
@@ -36,7 +36,9 @@ import (
 	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
-//var _ Interface = &Client{}
+var _ Interface = &Client{}
+
+const vmasResourceType = "Microsoft.Compute/availabilitySets"
 
 // Client implements VMAS client Interface.
 type Client struct {
@@ -120,7 +122,7 @@ func (c *Client) getVMAS(ctx context.Context, resourceGroupName string, vmasName
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/availabilitySets",
+		vmasResourceType,
 		vmasName,
 	)
 	result := compute.AvailabilitySet{}
@@ -178,7 +180,7 @@ func (c *Client) List(ctx context.Context, resourceGroupName string) ([]compute.
 
 // listVMAS gets a list of AvailabilitySets in the resource group.
 func (c *Client) listVMAS(ctx context.Context, resourceGroupName string) ([]compute.AvailabilitySet, *retry.Error) {
-	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, "Microsoft.Compute/availabilitySets")
+	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, vmasResourceType)
 	result := make([]compute.AvailabilitySet, 0)
 	page := &AvailabilitySetListResultPage{}
 	page.fn = c.listNextResults
diff --git a/pkg/azureclients/vmclient/azure_vmclient.go b/pkg/azureclients/vmclient/azure_vmclient.go
index ba89fb065..1856d1c60 100644
--- a/pkg/azureclients/vmclient/azure_vmclient.go
+++ b/pkg/azureclients/vmclient/azure_vmclient.go
@@ -38,6 +38,8 @@ import (
 
 var _ Interface = &Client{}
 
+const vmResourceType = "Microsoft.Compute/virtualMachines"
+
 // Client implements VirtualMachine client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -120,7 +122,7 @@ func (c *Client) getVM(ctx context.Context, resourceGroupName string, VMName str
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachines",
+		vmResourceType,
 		VMName,
 	)
 	result := compute.VirtualMachine{}
@@ -178,7 +180,7 @@ func (c *Client) List(ctx context.Context, resourceGroupName string) ([]compute.
 
 // listVM gets a list of VirtualMachines in the resourceGroupName.
 func (c *Client) listVM(ctx context.Context, resourceGroupName string) ([]compute.VirtualMachine, *retry.Error) {
-	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, "Microsoft.Compute/virtualMachines")
+	resourceID := armclient.GetResourceListID(c.subscriptionID, resourceGroupName, vmResourceType)
 
 	result := make([]compute.VirtualMachine, 0)
 	page := &VirtualMachineListResultPage{}
@@ -266,7 +268,7 @@ func (c *Client) UpdateAsync(ctx context.Context, resourceGroupName string, VMNa
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachines",
+		vmResourceType,
 		VMName,
 	)
 
@@ -306,7 +308,7 @@ func (c *Client) updateVM(ctx context.Context, resourceGroupName string, VMName
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachines",
+		vmResourceType,
 		VMName,
 	)
 
@@ -467,7 +469,7 @@ func (c *Client) createOrUpdateVM(ctx context.Context, resourceGroupName string,
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachines",
+		vmResourceType,
 		VMName,
 	)
 
@@ -536,7 +538,7 @@ func (c *Client) deleteVM(ctx context.Context, resourceGroupName string, VMName
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachines",
+		vmResourceType,
 		VMName,
 	)
 
diff --git a/pkg/azureclients/vmssclient/azure_vmssclient.go b/pkg/azureclients/vmssclient/azure_vmssclient.go
index c6b98d993..4a3a5c946 100644
--- a/pkg/azureclients/vmssclient/azure_vmssclient.go
+++ b/pkg/azureclients/vmssclient/azure_vmssclient.go
@@ -38,6 +38,8 @@ import (
 
 var _ Interface = &Client{}
 
+const vmssResourceType = "Microsoft.Compute/virtualMachineScaleSets"
+
 // Client implements VMSS client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -120,7 +122,7 @@ func (c *Client) getVMSS(ctx context.Context, resourceGroupName string, VMScaleS
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		VMScaleSetName,
 	)
 	result := compute.VirtualMachineScaleSet{}
@@ -181,7 +183,7 @@ func (c *Client) listVMSS(ctx context.Context, resourceGroupName string) ([]comp
 	resourceID := armclient.GetResourceListID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 	)
 	result := make([]compute.VirtualMachineScaleSet, 0)
 	page := &VirtualMachineScaleSetListResultPage{}
@@ -269,7 +271,7 @@ func (c *Client) CreateOrUpdateAsync(ctx context.Context, resourceGroupName stri
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		VMScaleSetName,
 	)
 
@@ -320,7 +322,7 @@ func (c *Client) createOrUpdateVMSS(ctx context.Context, resourceGroupName strin
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		VMScaleSetName,
 	)
 	response, rerr := c.armClient.PutResource(ctx, resourceID, parameters)
@@ -492,7 +494,7 @@ func (c *Client) DeleteInstancesAsync(ctx context.Context, resourceGroupName str
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		vmScaleSetName,
 	)
 
@@ -547,7 +549,7 @@ func (c *Client) DeallocateInstancesAsync(ctx context.Context, resourceGroupName
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		vmScaleSetName,
 	)
 
@@ -596,7 +598,7 @@ func (c *Client) StartInstancesAsync(ctx context.Context, resourceGroupName stri
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		vmScaleSetName,
 	)
 
@@ -630,7 +632,7 @@ func (c *Client) deleteVMSSInstances(ctx context.Context, resourceGroupName stri
 	resourceID := armclient.GetResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		vmScaleSetName,
 	)
 	response, rerr := c.armClient.PostResource(ctx, resourceID, "delete", vmInstanceIDs, map[string]interface{}{})
diff --git a/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go b/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go
index 7062439ea..04f47ac68 100644
--- a/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go
+++ b/pkg/azureclients/vmssvmclient/azure_vmssvmclient.go
@@ -39,6 +39,11 @@ import (
 
 var _ Interface = &Client{}
 
+const (
+	vmssResourceType = "Microsoft.Compute/virtualMachineScaleSets"
+	vmResourceType   = "virtualMachines"
+)
+
 // Client implements VMSS client Interface.
 type Client struct {
 	armClient      armclient.Interface
@@ -121,9 +126,9 @@ func (c *Client) getVMSSVM(ctx context.Context, resourceGroupName string, VMScal
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		VMScaleSetName,
-		"virtualMachines",
+		vmResourceType,
 		instanceID,
 	)
 	result := compute.VirtualMachineScaleSetVM{}
@@ -184,9 +189,9 @@ func (c *Client) listVMSSVM(ctx context.Context, resourceGroupName string, virtu
 	resourceID := armclient.GetChildResourcesListID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		virtualMachineScaleSetName,
-		"virtualMachines",
+		vmResourceType,
 	)
 
 	result := make([]compute.VirtualMachineScaleSetVM, 0)
@@ -275,9 +280,9 @@ func (c *Client) UpdateAsync(ctx context.Context, resourceGroupName string, VMSc
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		VMScaleSetName,
-		"virtualMachines",
+		vmResourceType,
 		instanceID,
 	)
 
@@ -316,9 +321,9 @@ func (c *Client) updateVMSSVM(ctx context.Context, resourceGroupName string, VMS
 	resourceID := armclient.GetChildResourceID(
 		c.subscriptionID,
 		resourceGroupName,
-		"Microsoft.Compute/virtualMachineScaleSets",
+		vmssResourceType,
 		VMScaleSetName,
-		"virtualMachines",
+		vmResourceType,
 		instanceID,
 	)
 
@@ -480,9 +485,9 @@ func (c *Client) updateVMSSVMs(ctx context.Context, resourceGroupName string, VM
 		resourceID := armclient.GetChildResourceID(
 			c.subscriptionID,
 			resourceGroupName,
-			"Microsoft.Compute/virtualMachineScaleSets",
+			vmssResourceType,
 			VMScaleSetName,
-			"virtualMachines",
+			vmResourceType,
 			instanceID,
 		)
 		resources[resourceID] = parameter
diff --git a/pkg/azureclients/zoneclient/azure_zoneclient.go b/pkg/azureclients/zoneclient/azure_zoneclient.go
index face94fcd..9a226d0b0 100644
--- a/pkg/azureclients/zoneclient/azure_zoneclient.go
+++ b/pkg/azureclients/zoneclient/azure_zoneclient.go
@@ -33,6 +33,8 @@ import (
 
 var _ Interface = &Client{}
 
+const computeResourceProvider = "Microsoft.Compute"
+
 type resourceTypeMetadata struct {
 	ResourceType string         `json:"resourceType"`
 	ZoneMappings []zoneMappings `json:"zoneMappings"`
@@ -113,7 +115,7 @@ func (c *Client) getZones(ctx context.Context, subscriptionID string) (map[strin
 	regionZoneMap := make(map[string][]string)
 	expectedID := armclient.GetProviderResourceID(
 		subscriptionID,
-		"Microsoft.Compute",
+		computeResourceProvider,
 	)
 	if len(result.ProviderListDataProperties) != 0 {
 		for _, property := range result.ProviderListDataProperties {
diff --git a/pkg/consts/helpers.go b/pkg/consts/helpers.go
index 44296f3d9..f86236218 100644
--- a/pkg/consts/helpers.go
+++ b/pkg/consts/helpers.go
@@ -23,7 +23,6 @@ import (
 	"strings"
 
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/utils/net"
 )
 
 // IsK8sServiceHasHAModeEnabled return if HA Mode is enabled in kuberntes service annotations
@@ -36,10 +35,6 @@ func IsK8sServiceUsingInternalLoadBalancer(service *v1.Service) bool {
 	return expectAttributeInSvcAnnotationBeEqualTo(service.Annotations, ServiceAnnotationLoadBalancerInternal, TrueAnnotationValue)
 }
 
-func IsK8sServiceInternalIPv6(service *v1.Service) bool {
-	return IsK8sServiceUsingInternalLoadBalancer(service) && net.IsIPv6String(service.Spec.ClusterIP)
-}
-
 // GetHealthProbeConfigOfPortFromK8sSvcAnnotation get health probe configuration for port
 func GetHealthProbeConfigOfPortFromK8sSvcAnnotation(annotations map[string]string, port int32, key HealthProbeParams, validators ...BusinessValidator) (*string, error) {
 	return GetAttributeValueInSvcAnnotation(annotations, BuildHealthProbeAnnotationKeyForPort(port, key), validators...)
diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go
index 909b340be..62988e924 100644
--- a/pkg/provider/azure_loadbalancer.go
+++ b/pkg/provider/azure_loadbalancer.go
@@ -2165,7 +2165,7 @@ func (az *Cloud) getExpectedLBRules(
 			if err != nil {
 				return expectedProbes, expectedRules, fmt.Errorf("failed to parse transport protocol: %w", err)
 			}
-			props, err := az.getExpectedLoadBalancingRulePropertiesForPort(service, lbFrontendIPConfigID, lbBackendPoolID, port, *transportProto)
+			props, err := az.getExpectedLoadBalancingRulePropertiesForPort(service, lbFrontendIPConfigID, lbBackendPoolID, to.Int32Ptr(port.Port), *transportProto)
 			if err != nil {
 				return expectedProbes, expectedRules, fmt.Errorf("error generate lb rule for ha mod loadbalancer. err: %w", err)
 			}
@@ -2203,7 +2203,7 @@ func (az *Cloud) getExpectedLBRules(
 func (az *Cloud) getExpectedLoadBalancingRulePropertiesForPort(
 	service *v1.Service,
 	lbFrontendIPConfigID string,
-	lbBackendPoolID string, servicePort v1.ServicePort, transportProto network.TransportProtocol) (*network.LoadBalancingRulePropertiesFormat, error) {
+	lbBackendPoolID string, port *int32, transportProto network.TransportProtocol) (*network.LoadBalancingRulePropertiesFormat, error) {
 	var err error
 
 	loadDistribution := network.LoadDistributionDefault
@@ -2229,8 +2229,8 @@ func (az *Cloud) getExpectedLoadBalancingRulePropertiesForPort(
 
 	props := &network.LoadBalancingRulePropertiesFormat{
 		Protocol:            transportProto,
-		FrontendPort:        to.Int32Ptr(servicePort.Port),
-		BackendPort:         to.Int32Ptr(servicePort.Port),
+		FrontendPort:        port,
+		BackendPort:         port,
 		DisableOutboundSnat: to.BoolPtr(az.disableLoadBalancerOutboundSNAT()),
 		EnableFloatingIP:    to.BoolPtr(true),
 		LoadDistribution:    loadDistribution,
@@ -2245,13 +2245,6 @@ func (az *Cloud) getExpectedLoadBalancingRulePropertiesForPort(
 	if strings.EqualFold(string(transportProto), string(network.TransportProtocolTCP)) && az.useStandardLoadBalancer() {
 		props.EnableTCPReset = to.BoolPtr(true)
 	}
-
-	// Azure ILB does not support secondary IPs as floating IPs on the LB. Therefore, floating IP needs to be turned
-	// off and the rule should point to the nodeIP:nodePort.
-	if consts.IsK8sServiceInternalIPv6(service) {
-		props.BackendPort = to.Int32Ptr(servicePort.NodePort)
-		props.EnableFloatingIP = to.BoolPtr(false)
-	}
 	return props, nil
 }
 
@@ -2260,7 +2253,7 @@ func (az *Cloud) getExpectedHAModeLoadBalancingRuleProperties(
 	service *v1.Service,
 	lbFrontendIPConfigID string,
 	lbBackendPoolID string) (*network.LoadBalancingRulePropertiesFormat, error) {
-	props, err := az.getExpectedLoadBalancingRulePropertiesForPort(service, lbFrontendIPConfigID, lbBackendPoolID, v1.ServicePort{}, network.TransportProtocolAll)
+	props, err := az.getExpectedLoadBalancingRulePropertiesForPort(service, lbFrontendIPConfigID, lbBackendPoolID, to.Int32Ptr(0), network.TransportProtocolAll)
 	if err != nil {
 		return nil, fmt.Errorf("error generate lb rule for ha mod loadbalancer. err: %w", err)
 	}
diff --git a/pkg/provider/azure_routes.go b/pkg/provider/azure_routes.go
index 53a9f655c..a669e1be8 100644
--- a/pkg/provider/azure_routes.go
+++ b/pkg/provider/azure_routes.go
@@ -383,6 +383,10 @@ func (az *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint s
 		return err
 	}
 	if unmanaged {
+		if az.ipv6DualStackEnabled {
+			//TODO (khenidak) add support for unmanaged nodes when the feature reaches beta
+			return fmt.Errorf("unmanaged nodes are not supported in dual stack mode")
+		}
 		klog.V(2).Infof("CreateRoute: omitting unmanaged node %q", kubeRoute.TargetNode)
 		az.routeCIDRsLock.Lock()
 		defer az.routeCIDRsLock.Unlock()
diff --git a/pkg/provider/azure_storageaccount.go b/pkg/provider/azure_storageaccount.go
index c31e027dd..a60bcd06b 100644
--- a/pkg/provider/azure_storageaccount.go
+++ b/pkg/provider/azure_storageaccount.go
@@ -360,12 +360,12 @@ func (az *Cloud) createPrivateDNSZone(ctx context.Context, vnetResourceGroup str
 	klog.V(2).Infof("Creating private dns zone(%s) in resourceGroup (%s)", PrivateDNSZoneName, vnetResourceGroup)
 	location := LocationGlobal
 	privateDNSZone := privatedns.PrivateZone{Location: &location}
-	if err := az.privatednsclient.CreateOrUpdate(ctx, vnetResourceGroup, PrivateDNSZoneName, privateDNSZone, true); err != nil {
-		if strings.Contains(err.Error(), "exists already") {
+	if err := az.privatednsclient.CreateOrUpdate(ctx, vnetResourceGroup, PrivateDNSZoneName, privateDNSZone, "", true); err != nil {
+		if strings.Contains(err.Error().Error(), "exists already") {
 			klog.V(2).Infof("private dns zone(%s) in resourceGroup (%s) already exists", PrivateDNSZoneName, vnetResourceGroup)
 			return nil
 		}
-		return err
+		return err.Error()
 	}
 	return nil
 }
@@ -398,7 +398,7 @@ func (az *Cloud) createPrivateDNSZoneGroup(ctx context.Context, dnsZoneGroupName
 			PrivateDNSZoneConfigs: &privateDNSZoneConfigs,
 		},
 	}
-	return az.privatednszonegroupclient.CreateOrUpdate(ctx, vnetResourceGroup, privateEndpointName, dnsZoneGroupName, privateDNSZoneGroup, false)
+	return az.privatednszonegroupclient.CreateOrUpdate(ctx, vnetResourceGroup, privateEndpointName, dnsZoneGroupName, privateDNSZoneGroup, "", false).Error()
 }
 
 // AddStorageAccountTags add tags to storage account


Test patch:
diff --git a/pkg/azureclients/privatednsclient/azure_privatednsclient_test.go b/pkg/azureclients/privatednsclient/azure_privatednsclient_test.go
new file mode 100644
index 000000000..7c498ecd8
--- /dev/null
+++ b/pkg/azureclients/privatednsclient/azure_privatednsclient_test.go
@@ -0,0 +1,272 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package privatednsclient
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/to"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/client-go/util/flowcontrol"
+
+	azclients "sigs.k8s.io/cloud-provider-azure/pkg/azureclients"
+	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient"
+	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient/mockarmclient"
+	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
+)
+
+const (
+	sub0 = "sub0"
+	rg0  = "rg0"
+	pz0  = "pz0"
+
+	testResourceID     = "/subscriptions/sub0/resourceGroups/rg0/providers/" + privateDNSZoneResourceType + "/" + pz0
+	testResourcePrefix = "/subscriptions/sub0/resourceGroups/rg0/providers/" + privateDNSZoneResourceType
+)
+
+func TestNew(t *testing.T) {
+	config := &azclients.ClientConfig{
+		SubscriptionID:          sub0,
+		ResourceManagerEndpoint: "endpoint",
+		Location:                "eastus",
+		RateLimitConfig: &azclients.RateLimitConfig{
+			CloudProviderRateLimit:            true,
+			CloudProviderRateLimitQPS:         0.5,
+			CloudProviderRateLimitBucket:      1,
+			CloudProviderRateLimitQPSWrite:    0.5,
+			CloudProviderRateLimitBucketWrite: 1,
+		},
+		Backoff: &retry.Backoff{Steps: 1},
+	}
+
+	pzClient := New(config)
+	assert.Equal(t, sub0, pzClient.subscriptionID)
+	assert.NotEmpty(t, pzClient.rateLimiterReader)
+	assert.NotEmpty(t, pzClient.rateLimiterWriter)
+}
+
+func TestCreateOrUpdate(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	pz := getTestPrivateDNSZone(pz0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
+	}
+	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(pz.ID), pz, gomock.Any()).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzClient := getTestPrivateDNSZoneClient(armClient)
+	rerr := pzClient.CreateOrUpdate(context.TODO(), rg0, pz0, pz, "", true)
+	assert.Nil(t, rerr)
+}
+
+func TestCreateOrUpdateWithNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	rcCreateOrUpdateErr := retry.GetRateLimitError(true, "PrivateDNSZoneCreateOrUpdate")
+
+	pz := getTestPrivateDNSZone(pz0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+
+	pzClient := getTestPrivateDNSZoneClientWithNeverRateLimiter(armClient)
+	rerr := pzClient.CreateOrUpdate(context.TODO(), rg0, pz0, pz, "", true)
+	assert.Equal(t, rcCreateOrUpdateErr, rerr)
+}
+
+func TestCreateOrUpdateRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	rcCreateOrUpdateErr := retry.GetThrottlingError("PrivateDNSZoneCreateOrUpdate", "client throttled", getFutureTime())
+
+	pz := getTestPrivateDNSZone(pz0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+
+	pzClient := getTestPrivateDNSZoneClientWithRetryAfterReader(armClient)
+	rerr := pzClient.CreateOrUpdate(context.TODO(), rg0, pz0, pz, "", true)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, rcCreateOrUpdateErr, rerr)
+}
+
+func TestCreateOrUpdateThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+
+	pz := getTestPrivateDNSZone(pz0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(pz.ID), pz, gomock.Any()).Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzClient := getTestPrivateDNSZoneClient(armClient)
+	rerr := pzClient.CreateOrUpdate(context.TODO(), rg0, pz0, pz, "", true)
+	assert.Equal(t, throttleErr, rerr)
+}
+
+func TestCreateOrUpdateWithCreateOrUpdateResponderError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	pz := getTestPrivateDNSZone(pz0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
+	}
+
+	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(pz.ID), pz, gomock.Any()).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzClient := getTestPrivateDNSZoneClient(armClient)
+	rerr := pzClient.CreateOrUpdate(context.TODO(), rg0, pz0, pz, "", true)
+	assert.NotNil(t, rerr)
+}
+
+func getTestPrivateDNSZone(name string) privatedns.PrivateZone {
+	return privatedns.PrivateZone{
+		ID:       to.StringPtr(fmt.Sprintf("%s/%s", testResourcePrefix, name)),
+		Name:     to.StringPtr(name),
+		Location: to.StringPtr("eastus"),
+	}
+}
+
+func getTestPrivateDNSZoneClient(armClient armclient.Interface) *Client {
+	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(&azclients.RateLimitConfig{})
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    sub0,
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+	}
+}
+
+func getTestPrivateDNSZoneClientWithNeverRateLimiter(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    sub0,
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+	}
+}
+
+func getTestPrivateDNSZoneClientWithRetryAfterReader(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    sub0,
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+		RetryAfterReader:  getFutureTime(),
+		RetryAfterWriter:  getFutureTime(),
+	}
+}
+
+func TestGetNotFound(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), testResourceID).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzClient := getTestPrivateDNSZoneClient(armClient)
+	expected := privatedns.PrivateZone{Response: autorest.Response{}}
+	result, rerr := pzClient.Get(context.TODO(), rg0, pz0)
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode)
+}
+
+func TestGetInternalError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusInternalServerError,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), testResourceID).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzClient := getTestPrivateDNSZoneClient(armClient)
+	expected := privatedns.PrivateZone{Response: autorest.Response{}}
+	result, rerr := pzClient.Get(context.TODO(), rg0, pz0)
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode)
+}
+
+func TestGetThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), testResourceID).Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzClient := getTestPrivateDNSZoneClient(armClient)
+	result, rerr := pzClient.Get(context.TODO(), rg0, pz0)
+	assert.Empty(t, result)
+	assert.Equal(t, throttleErr, rerr)
+}
+
+// 2065-01-24 05:20:00 +0000 UTC
+func getFutureTime() time.Time {
+	return time.Unix(3000000000, 0)
+}
diff --git a/pkg/azureclients/privatednszonegroupclient/azure_privatednszonegroupclient_test.go b/pkg/azureclients/privatednszonegroupclient/azure_privatednszonegroupclient_test.go
new file mode 100644
index 000000000..e3406b32a
--- /dev/null
+++ b/pkg/azureclients/privatednszonegroupclient/azure_privatednszonegroupclient_test.go
@@ -0,0 +1,272 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package privatednszonegroupclient
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/to"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/client-go/util/flowcontrol"
+
+	azclients "sigs.k8s.io/cloud-provider-azure/pkg/azureclients"
+	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient"
+	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient/mockarmclient"
+	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
+)
+
+const (
+	sub0 = "sub0"
+	rg0  = "rg0"
+	pe0  = "pe0"
+	pzg0 = "pzg0"
+
+	testResourceID     = "/subscriptions/sub0/resourceGroups/rg0/providers/Microsoft.Network/privateEndpoints/pe0/privateDnsZoneGroups/" + pzg0
+	testResourcePrefix = "/subscriptions/sub0/resourceGroups/rg0/providers/Microsoft.Network/privateEndpoints/pe0/privateDnsZoneGroups"
+)
+
+func TestNew(t *testing.T) {
+	config := &azclients.ClientConfig{
+		SubscriptionID:          sub0,
+		ResourceManagerEndpoint: "endpoint",
+		Location:                "eastus",
+		RateLimitConfig: &azclients.RateLimitConfig{
+			CloudProviderRateLimit:            true,
+			CloudProviderRateLimitQPS:         0.5,
+			CloudProviderRateLimitBucket:      1,
+			CloudProviderRateLimitQPSWrite:    0.5,
+			CloudProviderRateLimitBucketWrite: 1,
+		},
+		Backoff: &retry.Backoff{Steps: 1},
+	}
+
+	pzgClient := New(config)
+	assert.Equal(t, sub0, pzgClient.subscriptionID)
+	assert.NotEmpty(t, pzgClient.rateLimiterReader)
+	assert.NotEmpty(t, pzgClient.rateLimiterWriter)
+}
+
+func TestCreateOrUpdate(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	pzg := getTestPrivateDNSZoneGroup(pzg0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	response := &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
+	}
+	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(pzg.ID), pzg, gomock.Any()).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzgClient := getTestPrivateDNSZoneGroupClient(armClient)
+	rerr := pzgClient.CreateOrUpdate(context.TODO(), rg0, pe0, pzg0, pzg, "", true)
+	assert.Nil(t, rerr)
+}
+
+func TestCreateOrUpdateWithNeverRateLimiter(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	rcCreateOrUpdateErr := retry.GetRateLimitError(true, "PrivateDNSZoneGroupCreateOrUpdate")
+
+	pzg := getTestPrivateDNSZoneGroup(pzg0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+
+	pzgClient := getTestPrivateDNSZoneGroupClientWithNeverRateLimiter(armClient)
+	rerr := pzgClient.CreateOrUpdate(context.TODO(), rg0, pe0, pzg0, pzg, "", true)
+	assert.Equal(t, rcCreateOrUpdateErr, rerr)
+}
+
+func TestCreateOrUpdateRetryAfterReader(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	rcCreateOrUpdateErr := retry.GetThrottlingError("PrivateDNSZoneGroupCreateOrUpdate", "client throttled", getFutureTime())
+
+	pzg := getTestPrivateDNSZoneGroup(pzg0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+
+	pzgClient := getTestPrivateDNSZoneGroupClientWithRetryAfterReader(armClient)
+	rerr := pzgClient.CreateOrUpdate(context.TODO(), rg0, pe0, pzg0, pzg, "", true)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, rcCreateOrUpdateErr, rerr)
+}
+
+func TestCreateOrUpdateThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+
+	pzg := getTestPrivateDNSZoneGroup(pzg0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(pzg.ID), pzg, gomock.Any()).Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzgClient := getTestPrivateDNSZoneGroupClient(armClient)
+	rerr := pzgClient.CreateOrUpdate(context.TODO(), rg0, pe0, pzg0, pzg, "", true)
+	assert.Equal(t, throttleErr, rerr)
+}
+
+func TestCreateOrUpdateWithCreateOrUpdateResponderError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	pzg := getTestPrivateDNSZoneGroup(pzg0)
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
+	}
+
+	armClient.EXPECT().PutResourceWithDecorators(gomock.Any(), to.String(pzg.ID), pzg, gomock.Any()).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzgClient := getTestPrivateDNSZoneGroupClient(armClient)
+	rerr := pzgClient.CreateOrUpdate(context.TODO(), rg0, pe0, pzg0, pzg, "", true)
+	assert.NotNil(t, rerr)
+}
+
+func getTestPrivateDNSZoneGroup(name string) network.PrivateDNSZoneGroup {
+	return network.PrivateDNSZoneGroup{
+		ID:   to.StringPtr(fmt.Sprintf("%s/%s", testResourcePrefix, name)),
+		Name: to.StringPtr(name),
+	}
+}
+
+func getTestPrivateDNSZoneGroupClient(armClient armclient.Interface) *Client {
+	rateLimiterReader, rateLimiterWriter := azclients.NewRateLimiter(&azclients.RateLimitConfig{})
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    sub0,
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+	}
+}
+
+func getTestPrivateDNSZoneGroupClientWithNeverRateLimiter(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeNeverRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeNeverRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    sub0,
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+	}
+}
+
+func getTestPrivateDNSZoneGroupClientWithRetryAfterReader(armClient armclient.Interface) *Client {
+	rateLimiterReader := flowcontrol.NewFakeAlwaysRateLimiter()
+	rateLimiterWriter := flowcontrol.NewFakeAlwaysRateLimiter()
+	return &Client{
+		armClient:         armClient,
+		subscriptionID:    sub0,
+		rateLimiterReader: rateLimiterReader,
+		rateLimiterWriter: rateLimiterWriter,
+		RetryAfterReader:  getFutureTime(),
+		RetryAfterWriter:  getFutureTime(),
+	}
+}
+
+func TestGetNotFound(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusNotFound,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), testResourceID).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzgClient := getTestPrivateDNSZoneGroupClient(armClient)
+	expected := network.PrivateDNSZoneGroup{Response: autorest.Response{}}
+	result, rerr := pzgClient.Get(context.TODO(), rg0, pe0, pzg0)
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, http.StatusNotFound, rerr.HTTPStatusCode)
+}
+
+func TestGetInternalError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusInternalServerError,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), testResourceID).Return(response, nil).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzgClient := getTestPrivateDNSZoneGroupClient(armClient)
+	expected := network.PrivateDNSZoneGroup{Response: autorest.Response{}}
+	result, rerr := pzgClient.Get(context.TODO(), rg0, pe0, pzg0)
+	assert.Equal(t, expected, result)
+	assert.NotNil(t, rerr)
+	assert.Equal(t, http.StatusInternalServerError, rerr.HTTPStatusCode)
+}
+
+func TestGetThrottle(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	response := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Body:       ioutil.NopCloser(bytes.NewReader([]byte("{}"))),
+	}
+	throttleErr := &retry.Error{
+		HTTPStatusCode: http.StatusTooManyRequests,
+		RawError:       fmt.Errorf("error"),
+		Retriable:      true,
+		RetryAfter:     time.Unix(100, 0),
+	}
+	armClient := mockarmclient.NewMockInterface(ctrl)
+	armClient.EXPECT().GetResource(gomock.Any(), testResourceID).Return(response, throttleErr).Times(1)
+	armClient.EXPECT().CloseResponse(gomock.Any(), gomock.Any()).Times(1)
+
+	pzgClient := getTestPrivateDNSZoneGroupClient(armClient)
+	result, rerr := pzgClient.Get(context.TODO(), rg0, pe0, pzg0)
+	assert.Empty(t, result)
+	assert.Equal(t, throttleErr, rerr)
+}
+
+// 2065-01-24 05:20:00 +0000 UTC
+func getFutureTime() time.Time {
+	return time.Unix(3000000000, 0)
+}
diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go
index d7d098a20..10297396a 100644
--- a/pkg/provider/azure_loadbalancer_test.go
+++ b/pkg/provider/azure_loadbalancer_test.go
@@ -1880,15 +1880,6 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
 			expectedProbes:  getDefaultTestProbes("Https", "/healthy1"),
 			expectedRules:   getDefaultTestRules(true),
 		},
-		{
-			desc: "getExpectedLBRules shall return corresponding probe and lbRule (slb with IPv6)",
-			service: getTestService("test1", v1.ProtocolTCP, map[string]string{
-				"service.beta.kubernetes.io/azure-load-balancer-internal": "true",
-			}, true, 80),
-			loadBalancerSku: "standard",
-			expectedProbes:  getDefaultTestProbes("Tcp", ""),
-			expectedRules:   getDefaultInternalIPv6Rules(true),
-		},
 		{
 			desc: "getExpectedLBRules shall return corresponding probe and lbRule (slb with HA enabled)",
 			service: getTestService("test1", v1.ProtocolTCP, map[string]string{
@@ -2203,15 +2194,6 @@ func getDefaultTestRules(enableTCPReset bool) []network.LoadBalancingRule {
 	}
 }
 
-func getDefaultInternalIPv6Rules(enableTCPReset bool) []network.LoadBalancingRule {
-	rules := getDefaultTestRules(true)
-	for _, rule := range rules {
-		rule.EnableFloatingIP = to.BoolPtr(false)
-		rule.BackendPort = to.Int32Ptr(getBackendPort(*rule.FrontendPort))
-	}
-	return rules
-}
-
 func getTestRule(enableTCPReset bool, port int32) network.LoadBalancingRule {
 	expectedRules := network.LoadBalancingRule{
 		Name: to.StringPtr(fmt.Sprintf("atest1-TCP-%d", port)),
diff --git a/pkg/provider/azure_routes_test.go b/pkg/provider/azure_routes_test.go
index 7a765f80f..1d4e3776e 100644
--- a/pkg/provider/azure_routes_test.go
+++ b/pkg/provider/azure_routes_test.go
@@ -308,6 +308,13 @@ func TestCreateRoute(t *testing.T) {
 			routeCIDRs:         map[string]string{},
 			expectedRouteCIDRs: map[string]string{"node": "1.2.3.4/24"},
 		},
+		{
+			name:                 "CreateRoute should report error if node is unmanaged and cloud.ipv6DualStackEnabled is true",
+			hasUnmanagedNodes:    true,
+			ipv6DualStackEnabled: true,
+			unmanagedNodeName:    "node",
+			expectedErrMsg:       fmt.Errorf("unmanaged nodes are not supported in dual stack mode"),
+		},
 		{
 			name:           "CreateRoute should create route if cloud.ipv6DualStackEnabled is true and route doesn't exist",
 			routeTableName: "rt9",
diff --git a/pkg/provider/azure_storageaccount_test.go b/pkg/provider/azure_storageaccount_test.go
index d844548c6..392535669 100644
--- a/pkg/provider/azure_storageaccount_test.go
+++ b/pkg/provider/azure_storageaccount_test.go
@@ -445,11 +445,11 @@ func TestEnsureStorageAccount(t *testing.T) {
 			cloud.SubnetsClient = mockSubnetsClient
 
 			mockPrivateDNSClient := mockprivatednsclient.NewMockInterface(ctrl)
-			mockPrivateDNSClient.EXPECT().CreateOrUpdate(gomock.Any(), vnetResourceGroup, gomock.Any(), gomock.Any(), true).Return(nil).Times(1)
+			mockPrivateDNSClient.EXPECT().CreateOrUpdate(gomock.Any(), vnetResourceGroup, gomock.Any(), gomock.Any(), "", true).Return(nil).Times(1)
 			cloud.privatednsclient = mockPrivateDNSClient
 
 			mockPrivateDNSZoneGroup := mockprivatednszonegroupclient.NewMockInterface(ctrl)
-			mockPrivateDNSZoneGroup.EXPECT().CreateOrUpdate(gomock.Any(), vnetResourceGroup, gomock.Any(), gomock.Any(), gomock.Any(), false).Return(nil).Times(1)
+			mockPrivateDNSZoneGroup.EXPECT().CreateOrUpdate(gomock.Any(), vnetResourceGroup, gomock.Any(), gomock.Any(), gomock.Any(), "", false).Return(nil).Times(1)
 			cloud.privatednszonegroupclient = mockPrivateDNSZoneGroup
 
 			mockPrivateEndpointClient := mockprivateendpointclient.NewMockInterface(ctrl)