Repository: kubernetes-sigs/cloud-provider-azure
Pull request #2835 (2022-11-30 02:21:25)

fix: race condition in a unit test
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?

<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

/kind failing-test

#### What this PR does / why we need it:

fix: race condition in a unit test

#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
NONE
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```



Patch:
diff --git a/go.mod b/go.mod
index 41fcfb123..5c4c3d864 100644
--- a/go.mod
+++ b/go.mod
@@ -19,6 +19,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.1
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	golang.org/x/sys v0.2.0
 	golang.org/x/text v0.4.0
 	k8s.io/api v0.25.4
@@ -100,7 +101,6 @@ require (
 	go.uber.org/zap v1.19.0 // indirect
 	golang.org/x/net v0.2.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	golang.org/x/term v0.2.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
diff --git a/pkg/provider/azure_utils.go b/pkg/provider/azure_utils.go
index fe353f708..910abb56f 100644
--- a/pkg/provider/azure_utils.go
+++ b/pkg/provider/azure_utils.go
@@ -298,40 +298,36 @@ func removeDuplicatedSecurityRules(rules []network.SecurityRule) []network.Secur
 	return rules
 }
 
-func getVMSSVMCacheKey(resourceGroup, vmssName string) string {
-	cacheKey := strings.ToLower(fmt.Sprintf("%s/%s", resourceGroup, vmssName))
-	return cacheKey
-}
-
 // isNodeInVMSSVMCache check whether nodeName is in vmssVMCache
-func isNodeInVMSSVMCache(nodeName string, vmssVMCache *azcache.TimedCache) bool {
+func isNodeInVMSSVMCache(nodeName string, vmssVMCache *sync.Map) bool {
 	if vmssVMCache == nil {
 		return false
 	}
-
 	var isInCache bool
-
-	vmssVMCache.Lock.Lock()
-	defer vmssVMCache.Lock.Unlock()
-
-	for _, entry := range vmssVMCache.Store.List() {
-		if entry != nil {
-			data := entry.(*azcache.AzureCacheEntry).Data
-			if data != nil {
-				data.(*sync.Map).Range(func(vmName, _ interface{}) bool {
-					if vmName != nil && vmName.(string) == nodeName {
-						isInCache = true
-						return false
+	vmssVMCache.Range(func(_, value interface{}) bool {
+		if value != nil && value.(*azcache.TimedCache).Store != nil {
+			for _, v := range value.(*azcache.TimedCache).Store.List() {
+				if v != nil {
+					data := v.(*azcache.AzureCacheEntry).Data
+					if data != nil {
+						data.(*sync.Map).Range(func(vmName, _ interface{}) bool {
+							if vmName != nil && vmName.(string) == nodeName {
+								isInCache = true
+								return false
+							}
+							return true
+						})
 					}
-					return true
-				})
+				}
+				if isInCache {
+					return false
+				}
 			}
 		}
-
 		if isInCache {
-			break
+			return false
 		}
-	}
-
+		return true
+	})
 	return isInCache
 }
diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go
index 43ce1bf4f..cd56e252a 100644
--- a/pkg/provider/azure_vmss.go
+++ b/pkg/provider/azure_vmss.go
@@ -35,6 +35,7 @@ import (
 	"k8s.io/klog/v2"
 	utilnet "k8s.io/utils/net"
 
+	"golang.org/x/sync/singleflight"
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
@@ -79,12 +80,8 @@ type ScaleSet struct {
 	// the same cluster.
 	flexScaleSet VMSet
 
-	vmssCache *azcache.TimedCache
-
-	// vmssVMCache is timed cache where the Store in the cache is a map of
-	// Key: [resourcegroup/vmssName]
-	// Value: sync.Map of [vmName]*VMSSVirtualMachinesEntry
-	vmssVMCache *azcache.TimedCache
+	vmssCache   *azcache.TimedCache
+	vmssVMCache *sync.Map // [resourcegroup/vmssname]*azcache.TimedCache
 
 	// nonVmssUniformNodesCache is used to store node names from non uniform vm.
 	// Currently, the nodes can from avset or vmss flex or individual vm.
@@ -95,6 +92,8 @@ type ScaleSet struct {
 
 	// lockMap in cache refresh
 	lockMap *lockMap
+	// group represents a class of work and units of work can be executed with duplicate suppression
+	group singleflight.Group
 }
 
 // newScaleSet creates a new ScaleSet.
@@ -117,6 +116,7 @@ func newScaleSet(az *Cloud) (VMSet, error) {
 		Cloud:           az,
 		availabilitySet: as,
 		flexScaleSet:    fs,
+		vmssVMCache:     &sync.Map{},
 		lockMap:         newLockMap(),
 	}
 
@@ -132,11 +132,6 @@ func newScaleSet(az *Cloud) (VMSet, error) {
 		return nil, err
 	}
 
-	ss.vmssVMCache, err = ss.newVMSSVirtualMachinesCache()
-	if err != nil {
-		return nil, err
-	}
-
 	return ss, nil
 }
 
@@ -179,23 +174,22 @@ func (ss *ScaleSet) getVMSS(vmssName string, crt azcache.AzureCacheReadType) (*c
 // getVmssVMByNodeIdentity find virtualMachineScaleSetVM by nodeIdentity, using node's parent VMSS cache.
 // Returns cloudprovider.InstanceNotFound if the node does not belong to the scale set named in nodeIdentity.
 func (ss *ScaleSet) getVmssVMByNodeIdentity(node *nodeIdentity, crt azcache.AzureCacheReadType) (*virtualmachine.VirtualMachine, error) {
-	// FIXME(ccc): check only if vmss is uniform.
-	_, err := getScaleSetVMInstanceID(node.nodeName)
+	cacheKey, cache, err := ss.getVMSSVMCache(node.resourceGroup, node.vmssName)
 	if err != nil {
 		return nil, err
 	}
 
-	getter := func(crt azcache.AzureCacheReadType) (*virtualmachine.VirtualMachine, bool, error) {
+	getter := func(nodeName string, crt azcache.AzureCacheReadType) (*virtualmachine.VirtualMachine, bool, error) {
 		var found bool
-		virtualMachines, err := ss.getVMSSVMsFromCache(node.resourceGroup, node.vmssName, crt)
+		cached, err := cache.Get(cacheKey, crt)
 		if err != nil {
 			return nil, found, err
 		}
-
-		if entry, ok := virtualMachines.Load(node.nodeName); ok {
+		virtualMachines := cached.(*sync.Map)
+		if entry, ok := virtualMachines.Load(nodeName); ok {
 			result := entry.(*VMSSVirtualMachinesEntry)
 			if result.VirtualMachine == nil {
-				klog.Warningf("VM is nil on Node %q, VM is in deleting state", node.nodeName)
+				klog.Warningf("VM is nil on Node %q, VM is in deleting state", nodeName)
 				return nil, true, nil
 			}
 			found = true
@@ -205,24 +199,29 @@ func (ss *ScaleSet) getVmssVMByNodeIdentity(node *nodeIdentity, crt azcache.Azur
 		return nil, found, nil
 	}
 
-	vm, found, err := getter(crt)
+	// FIXME(ccc): check only if vmss is uniform.
+	_, err = getScaleSetVMInstanceID(node.nodeName)
+	if err != nil {
+		return nil, err
+	}
+
+	vm, found, err := getter(node.nodeName, crt)
 	if err != nil {
 		return nil, err
 	}
 
 	if !found {
-		cacheKey := getVMSSVMCacheKey(node.resourceGroup, node.vmssName)
 		// lock and try find nodeName from cache again, refresh cache if still not found
 		ss.lockMap.LockEntry(cacheKey)
 		defer ss.lockMap.UnlockEntry(cacheKey)
-		vm, found, err = getter(crt)
+		vm, found, err = getter(node.nodeName, crt)
 		if err == nil && found && vm != nil {
 			klog.V(2).Infof("found VMSS VM with nodeName %s after retry", node.nodeName)
 			return vm, nil
 		}
 
 		klog.V(2).Infof("Couldn't find VMSS VM with nodeName %s, refreshing the cache(vmss: %s, rg: %s)", node.nodeName, node.vmssName, node.resourceGroup)
-		vm, found, err = getter(azcache.CacheReadTypeForceRefresh)
+		vm, found, err = getter(node.nodeName, azcache.CacheReadTypeForceRefresh)
 		if err != nil {
 			return nil, err
 		}
@@ -322,12 +321,18 @@ func (ss *ScaleSet) GetProvisioningStateByNodeName(name string) (provisioningSta
 // getCachedVirtualMachineByInstanceID gets scaleSetVMInfo from cache.
 // The node must belong to one of scale sets.
 func (ss *ScaleSet) getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID string, crt azcache.AzureCacheReadType) (*compute.VirtualMachineScaleSetVM, error) {
+	cacheKey, cache, err := ss.getVMSSVMCache(resourceGroup, scaleSetName)
+	if err != nil {
+		return nil, err
+	}
+
 	getter := func(crt azcache.AzureCacheReadType) (vm *compute.VirtualMachineScaleSetVM, found bool, err error) {
-		virtualMachines, err := ss.getVMSSVMsFromCache(resourceGroup, scaleSetName, crt)
+		cached, err := cache.Get(cacheKey, crt)
 		if err != nil {
 			return nil, false, err
 		}
 
+		virtualMachines := cached.(*sync.Map)
 		virtualMachines.Range(func(key, value interface{}) bool {
 			vmEntry := value.(*VMSSVirtualMachinesEntry)
 			if strings.EqualFold(vmEntry.ResourceGroup, resourceGroup) &&
diff --git a/pkg/provider/azure_vmss_cache.go b/pkg/provider/azure_vmss_cache.go
index 44ec739fc..62ec948e9 100644
--- a/pkg/provider/azure_vmss_cache.go
+++ b/pkg/provider/azure_vmss_cache.go
@@ -104,13 +104,19 @@ func (ss *ScaleSet) newVMSSCache() (*azcache.TimedCache, error) {
 
 		if resourceGroupNotFound {
 			// gc vmss vm cache when there is resource group not found
-			vmssVMKeys := ss.vmssVMCache.Store.ListKeys()
-			for _, cacheKey := range vmssVMKeys {
-				vmssName := cacheKey[strings.LastIndex(cacheKey, "/")+1:]
-				if _, ok := localCache.Load(vmssName); !ok {
-					klog.V(2).Infof("remove vmss %s from vmssVMCache due to rg not found", cacheKey)
-					_ = ss.vmssVMCache.Delete(cacheKey)
+			removed := map[string]bool{}
+			ss.vmssVMCache.Range(func(key, value interface{}) bool {
+				cacheKey := key.(string)
+				vlistIdx := cacheKey[strings.LastIndex(cacheKey, "/")+1:]
+				if _, ok := localCache.Load(vlistIdx); !ok {
+					klog.V(2).Infof("remove vmss %s from cache due to rg not found", cacheKey)
+					removed[cacheKey] = true
 				}
+				return true
+			})
+
+			for key := range removed {
+				ss.vmssVMCache.Delete(key)
 			}
 		}
 		return localCache, nil
@@ -136,20 +142,25 @@ func extractVmssVMName(name string) (string, string, error) {
 	return ssName, instanceID, nil
 }
 
-func (ss *ScaleSet) getVMSSVMsFromCache(resourceGroup, vmssName string, crt azcache.AzureCacheReadType) (*sync.Map, error) {
-	cacheKey := getVMSSVMCacheKey(resourceGroup, vmssName)
-	entry, err := ss.vmssVMCache.Get(cacheKey, crt)
-	if err != nil {
-		return nil, err
+// getVMSSVMCache returns an *azcache.TimedCache and cache key for a VMSS (creating that cache if new).
+func (ss *ScaleSet) getVMSSVMCache(resourceGroup, vmssName string) (string, *azcache.TimedCache, error) {
+	cacheKey := strings.ToLower(fmt.Sprintf("%s/%s", resourceGroup, vmssName))
+	if entry, ok := ss.vmssVMCache.Load(cacheKey); ok {
+		return cacheKey, entry.(*azcache.TimedCache), nil
 	}
 
-	if entry == nil {
-		err = fmt.Errorf("vmssVMCache entry for resourceGroup (%s), vmssName (%s) returned nil data", resourceGroup, vmssName)
-		return nil, err
+	v, err, _ := ss.group.Do(cacheKey, func() (interface{}, error) {
+		cache, err := ss.newVMSSVirtualMachinesCache(resourceGroup, vmssName, cacheKey)
+		if err != nil {
+			return nil, err
+		}
+		ss.vmssVMCache.Store(cacheKey, cache)
+		return cache, nil
+	})
+	if err != nil {
+		return "", nil, err
 	}
-
-	virtualMachines := entry.(*sync.Map)
-	return virtualMachines, nil
+	return cacheKey, v.(*azcache.TimedCache), nil
 }
 
 // gcVMSSVMCache delete stale VMSS VMs caches from deleted VMSSes.
@@ -158,37 +169,33 @@ func (ss *ScaleSet) gcVMSSVMCache() error {
 }
 
 // newVMSSVirtualMachinesCache instantiates a new VMs cache for VMs belonging to the provided VMSS.
-func (ss *ScaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) {
+func (ss *ScaleSet) newVMSSVirtualMachinesCache(resourceGroupName, vmssName, cacheKey string) (*azcache.TimedCache, error) {
 	vmssVirtualMachinesCacheTTL := time.Duration(ss.Config.VmssVirtualMachinesCacheTTLInSeconds) * time.Second
 
-	getter := func(cacheKey string) (interface{}, error) {
+	getter := func(key string) (interface{}, error) {
 		localCache := &sync.Map{} // [nodeName]*VMSSVirtualMachinesEntry
 
-		oldCache := make(map[string]*VMSSVirtualMachinesEntry)
+		oldCache := make(map[string]VMSSVirtualMachinesEntry)
 
-		entry, exists, err := ss.vmssVMCache.Store.GetByKey(cacheKey)
-		if err != nil {
-			return nil, err
-		}
-		if exists {
-			cached := entry.(*azcache.AzureCacheEntry).Data
-			if cached != nil {
-				virtualMachines := cached.(*sync.Map)
-				virtualMachines.Range(func(key, value interface{}) bool {
-					oldCache[key.(string)] = value.(*VMSSVirtualMachinesEntry)
-					return true
-				})
+		if vmssCache, ok := ss.vmssVMCache.Load(cacheKey); ok {
+			// get old cache before refreshing the cache
+			cache := vmssCache.(*azcache.TimedCache)
+			entry, exists, err := cache.Store.GetByKey(cacheKey)
+			if err != nil {
+				return nil, err
+			}
+			if exists {
+				cached := entry.(*azcache.AzureCacheEntry).Data
+				if cached != nil {
+					virtualMachines := cached.(*sync.Map)
+					virtualMachines.Range(func(key, value interface{}) bool {
+						oldCache[key.(string)] = *value.(*VMSSVirtualMachinesEntry)
+						return true
+					})
+				}
 			}
 		}
 
-		result := strings.Split(cacheKey, "/")
-		if len(result) < 2 {
-			err = fmt.Errorf("Invalid cacheKey (%s)", cacheKey)
-			return nil, err
-		}
-
-		resourceGroupName, vmssName := result[0], result[1]
-
 		vms, err := ss.listScaleSetVMs(vmssName, resourceGroupName)
 		if err != nil {
 			return nil, err
@@ -280,9 +287,15 @@ func (ss *ScaleSet) DeleteCacheForNode(nodeName string) error {
 		return err
 	}
 
-	err = ss.vmssVMCache.Delete(getVMSSVMCacheKey(node.resourceGroup, node.vmssName))
+	cacheKey, timedcache, err := ss.getVMSSVMCache(node.resourceGroup, node.vmssName)
 	if err != nil {
-		klog.Errorf("DeleteCacheForNode(%s) failed to remove from vmssVMCache with error: %v", nodeName, err)
+		klog.Errorf("DeleteCacheForNode(%s) failed with error: %v", nodeName, err)
+		return err
+	}
+
+	err = timedcache.Delete(cacheKey)
+	if err != nil {
+		klog.Errorf("DeleteCacheForNode(%s) failed with error: %v", nodeName, err)
 		return err
 	}
 
@@ -295,9 +308,9 @@ func (ss *ScaleSet) DeleteCacheForNode(nodeName string) error {
 }
 
 func (ss *ScaleSet) updateCache(nodeName, resourceGroupName, vmssName, instanceID string, updatedVM *compute.VirtualMachineScaleSetVM) error {
-	virtualMachines, err := ss.getVMSSVMsFromCache(resourceGroupName, vmssName, azcache.CacheReadTypeUnsafe)
+	cacheKey, timedCache, err := ss.getVMSSVMCache(resourceGroupName, vmssName)
 	if err != nil {
-		err = fmt.Errorf("updateCache(%s, %s, %s) failed getting vmCache with error: %v", vmssName, resourceGroupName, nodeName, err)
+		klog.Errorf("updateCache(%s, %s, %s) failed with error: %v", vmssName, resourceGroupName, nodeName, err)
 		return err
 	}
 
@@ -313,6 +326,14 @@ func (ss *ScaleSet) updateCache(nodeName, resourceGroupName, vmssName, instanceI
 
 	localCache.Store(nodeName, vmssVMCacheEntry)
 
+	vmCache, err := timedCache.Get(cacheKey, azcache.CacheReadTypeUnsafe)
+	if err != nil {
+		return err
+	}
+	if vmCache == nil {
+		return fmt.Errorf("nil vmCache")
+	}
+	virtualMachines := vmCache.(*sync.Map)
 	virtualMachines.Range(func(key, value interface{}) bool {
 		if key.(string) != nodeName {
 			localCache.Store(key.(string), value.(*VMSSVirtualMachinesEntry))
@@ -320,8 +341,7 @@ func (ss *ScaleSet) updateCache(nodeName, resourceGroupName, vmssName, instanceI
 		return true
 	})
 
-	cacheKey := getVMSSVMCacheKey(resourceGroupName, vmssName)
-	ss.vmssVMCache.Update(cacheKey, localCache)
+	timedCache.Update(cacheKey, localCache)
 	klog.V(4).Infof("updateCache(%s, %s, %s) for cacheKey(%s) updated successfully", vmssName, resourceGroupName, nodeName, cacheKey)
 	return nil
 }


Test patch:
diff --git a/pkg/provider/azure_controller_vmss_test.go b/pkg/provider/azure_controller_vmss_test.go
index 48b038c5c..d14779067 100644
--- a/pkg/provider/azure_controller_vmss_test.go
+++ b/pkg/provider/azure_controller_vmss_test.go
@@ -19,7 +19,6 @@ package provider
 import (
 	"fmt"
 	"net/http"
-	"strings"
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute"
@@ -219,7 +218,7 @@ func TestDetachDiskWithVMSS(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		scaleSetName := strings.ToLower(string(test.vmssName))
+		scaleSetName := string(test.vmssName)
 		ss, err := NewTestScaleSet(ctrl)
 		assert.NoError(t, err, test.desc)
 		testCloud := ss.cloud
@@ -265,7 +264,7 @@ func TestDetachDiskWithVMSS(t *testing.T) {
 		}
 		mockVMSSVMClient := testCloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 		mockVMSSVMClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup, scaleSetName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
-		if scaleSetName == strings.ToLower(string(fakeStatusNotFoundVMSSName)) {
+		if scaleSetName == string(fakeStatusNotFoundVMSSName) {
 			mockVMSSVMClient.EXPECT().Update(gomock.Any(), testCloud.ResourceGroup, scaleSetName, gomock.Any(), gomock.Any(), gomock.Any()).Return(updatedVMSSVM, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes()
 		} else {
 			mockVMSSVMClient.EXPECT().Update(gomock.Any(), testCloud.ResourceGroup, scaleSetName, gomock.Any(), gomock.Any(), gomock.Any()).Return(updatedVMSSVM, nil).AnyTimes()
@@ -349,7 +348,7 @@ func TestUpdateVMWithVMSS(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		scaleSetName := strings.ToLower(string(test.vmssName))
+		scaleSetName := string(test.vmssName)
 		ss, err := NewTestScaleSet(ctrl)
 		assert.NoError(t, err, test.desc)
 		testCloud := ss.cloud
@@ -386,7 +385,7 @@ func TestUpdateVMWithVMSS(t *testing.T) {
 		}
 		mockVMSSVMClient := testCloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 		mockVMSSVMClient.EXPECT().List(gomock.Any(), testCloud.ResourceGroup, scaleSetName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
-		if scaleSetName == strings.ToLower(string(fakeStatusNotFoundVMSSName)) {
+		if scaleSetName == string(fakeStatusNotFoundVMSSName) {
 			mockVMSSVMClient.EXPECT().Update(gomock.Any(), testCloud.ResourceGroup, scaleSetName, gomock.Any(), gomock.Any(), gomock.Any()).Return(updatedVMSSVM, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes()
 		} else {
 			mockVMSSVMClient.EXPECT().Update(gomock.Any(), testCloud.ResourceGroup, scaleSetName, gomock.Any(), gomock.Any(), gomock.Any()).Return(updatedVMSSVM, nil).AnyTimes()
diff --git a/pkg/provider/azure_utils_test.go b/pkg/provider/azure_utils_test.go
index 7308269cf..ff1a2c530 100644
--- a/pkg/provider/azure_utils_test.go
+++ b/pkg/provider/azure_utils_test.go
@@ -317,62 +317,29 @@ func TestRemoveDuplicatedSecurityRules(t *testing.T) {
 	}
 }
 
-func TestGetVMSSVMCacheKey(t *testing.T) {
-	tests := []struct {
-		description       string
-		resourceGroupName string
-		vmssName          string
-		cacheKey          string
-	}{
-		{
-			description:       "Resource group and Vmss Name are in lower case",
-			resourceGroupName: "resgrp",
-			vmssName:          "vmss",
-			cacheKey:          "resgrp/vmss",
-		},
-		{
-			description:       "Resource group has upper case and Vmss Name is in lower case",
-			resourceGroupName: "Resgrp",
-			vmssName:          "vmss",
-			cacheKey:          "resgrp/vmss",
-		},
-		{
-			description:       "Resource group is in lower case and Vmss Name has upper case",
-			resourceGroupName: "resgrp",
-			vmssName:          "Vmss",
-			cacheKey:          "resgrp/vmss",
-		},
-		{
-			description:       "Resource group and Vmss Name are both in upper case",
-			resourceGroupName: "Resgrp",
-			vmssName:          "Vmss",
-			cacheKey:          "resgrp/vmss",
-		},
-	}
-
-	for _, test := range tests {
-		result := getVMSSVMCacheKey(test.resourceGroupName, test.vmssName)
-		assert.Equal(t, result, test.cacheKey, test.description)
-	}
-}
-
 func TestIsNodeInVMSSVMCache(t *testing.T) {
+	emptyTimedCacheMap := &sync.Map{}
+	emptyTimedCacheMap.Store("key", &azcache.TimedCache{})
 
 	getter := func(key string) (interface{}, error) {
 		return nil, nil
 	}
 	emptyCacheEntryTimedCache, _ := azcache.NewTimedcache(fakeCacheTTL, getter)
 	emptyCacheEntryTimedCache.Set("key", nil)
+	emptyCacheEntryTimedCacheMap := &sync.Map{}
+	emptyCacheEntryTimedCacheMap.Store("key", emptyCacheEntryTimedCache)
 
 	cacheEntryTimedCache, _ := azcache.NewTimedcache(fakeCacheTTL, getter)
 	syncMap := &sync.Map{}
 	syncMap.Store("node", nil)
 	cacheEntryTimedCache.Set("key", syncMap)
+	cacheEntryTimedCacheMap := &sync.Map{}
+	cacheEntryTimedCacheMap.Store("key", cacheEntryTimedCache)
 
 	tests := []struct {
 		description    string
 		nodeName       string
-		vmssVMCache    *azcache.TimedCache
+		vmssVMCache    *sync.Map
 		expectedResult bool
 	}{
 		{
@@ -380,21 +347,31 @@ func TestIsNodeInVMSSVMCache(t *testing.T) {
 			vmssVMCache:    nil,
 			expectedResult: false,
 		},
+		{
+			description:    "empty cache",
+			vmssVMCache:    &sync.Map{},
+			expectedResult: false,
+		},
+		{
+			description:    "empty timed cache",
+			vmssVMCache:    emptyTimedCacheMap,
+			expectedResult: false,
+		},
 		{
 			description:    "empty CacheEntry timed cache",
-			vmssVMCache:    emptyCacheEntryTimedCache,
+			vmssVMCache:    emptyCacheEntryTimedCacheMap,
 			expectedResult: false,
 		},
 		{
 			description:    "node name in the cache",
 			nodeName:       "node",
-			vmssVMCache:    cacheEntryTimedCache,
+			vmssVMCache:    cacheEntryTimedCacheMap,
 			expectedResult: true,
 		},
 		{
 			description:    "node name not in the cache",
 			nodeName:       "node2",
-			vmssVMCache:    cacheEntryTimedCache,
+			vmssVMCache:    cacheEntryTimedCacheMap,
 			expectedResult: false,
 		},
 	}
diff --git a/pkg/provider/azure_vmss_cache_test.go b/pkg/provider/azure_vmss_cache_test.go
index 2404415b1..3895938b7 100644
--- a/pkg/provider/azure_vmss_cache_test.go
+++ b/pkg/provider/azure_vmss_cache_test.go
@@ -19,6 +19,7 @@ package provider
 import (
 	"fmt"
 	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute"
@@ -177,7 +178,7 @@ func TestVMSSVMCacheClearedWhenRGDeleted(t *testing.T) {
 	mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, nil).Times(1)
 
 	expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, testVMSSName, "", 0, vmList, "", false)
-	mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).Times(1)
+	mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, nil).AnyTimes()
 
 	// validate getting VMSS VM via cache.
 	vm := expectedVMs[0]
@@ -189,20 +190,19 @@ func TestVMSSVMCacheClearedWhenRGDeleted(t *testing.T) {
 	assert.Equal(t, &vm, realVM.AsVirtualMachineScaleSetVM())
 
 	// verify cache has test vmss.
-	cacheKey := getVMSSVMCacheKey("rg", testVMSSName)
-	_, err = ss.vmssVMCache.Get(cacheKey, azcache.CacheReadTypeDefault)
-	assert.Nil(t, err)
+	cacheKey := strings.ToLower(fmt.Sprintf("%s/%s", "rg", testVMSSName))
+	_, ok := ss.vmssVMCache.Load(cacheKey)
+	assert.Equal(t, true, ok)
 
 	// refresh the cache with error.
 	mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{}, &retry.Error{HTTPStatusCode: http.StatusNotFound}).Times(2)
-	mockVMSSVMClient.EXPECT().List(gomock.Any(), "rg", testVMSSName, gomock.Any()).Return([]compute.VirtualMachineScaleSetVM{}, &retry.Error{HTTPStatusCode: http.StatusNotFound}).Times(1)
 	realVM, err = ss.getVmssVM(vmName, azcache.CacheReadTypeForceRefresh)
 	assert.Nil(t, realVM)
 	assert.Equal(t, cloudprovider.InstanceNotFound, err)
 
 	// verify cache is cleared.
-	_, err = ss.vmssVMCache.Get(cacheKey, azcache.CacheReadTypeDefault)
-	assert.NotNil(t, err)
+	_, ok = ss.vmssVMCache.Load(cacheKey)
+	assert.Equal(t, false, ok)
 }
 
 func TestGetVMManagementTypeByNodeName(t *testing.T) {
diff --git a/pkg/provider/azure_vmss_test.go b/pkg/provider/azure_vmss_test.go
index 73917767a..e6159959b 100644
--- a/pkg/provider/azure_vmss_test.go
+++ b/pkg/provider/azure_vmss_test.go
@@ -19,6 +19,7 @@ package provider
 import (
 	"fmt"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute"
@@ -954,9 +955,12 @@ func TestGetVmssVMByNodeIdentity(t *testing.T) {
 			mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 			mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
 
-			cacheKey := getVMSSVMCacheKey(ss.ResourceGroup, testVMSSName)
-			virtualMachines, err := ss.getVMSSVMsFromCache(ss.ResourceGroup, testVMSSName, azcache.CacheReadTypeDefault)
+			// Make some nil VMSS VM in cache.
+			cacheKey, cache, err := ss.getVMSSVMCache(ss.ResourceGroup, testVMSSName)
 			assert.Nil(t, err)
+			cached, err := cache.Get(cacheKey, azcache.CacheReadTypeDefault)
+			assert.Nil(t, err)
+			virtualMachines := cached.(*sync.Map)
 			for _, vm := range test.goneVMList {
 				entry := VMSSVirtualMachinesEntry{
 					ResourceGroup: ss.ResourceGroup,
@@ -964,7 +968,7 @@ func TestGetVmssVMByNodeIdentity(t *testing.T) {
 				}
 				virtualMachines.Store(vm, &entry)
 			}
-			ss.vmssVMCache.Update(cacheKey, virtualMachines)
+			cache.Set(cacheKey, cached)
 
 			for i := 0; i < len(test.vmList); i++ {
 				node := nodeIdentity{ss.ResourceGroup, testVMSSName, test.vmList[i]}
@@ -978,9 +982,11 @@ func TestGetVmssVMByNodeIdentity(t *testing.T) {
 				assert.Equal(t, test.goneVMExpectedErr, err)
 			}
 
-			virtualMachines, err = ss.getVMSSVMsFromCache(ss.ResourceGroup, testVMSSName, azcache.CacheReadTypeDefault)
+			cacheKey, cache, err = ss.getVMSSVMCache(ss.ResourceGroup, testVMSSName)
 			assert.Nil(t, err)
-
+			cached, err = cache.Get(cacheKey, azcache.CacheReadTypeDefault)
+			assert.Nil(t, err)
+			virtualMachines = cached.(*sync.Map)
 			for _, vm := range test.goneVMList {
 				_, ok := virtualMachines.Load(vm)
 				assert.True(t, ok)
@@ -2673,7 +2679,7 @@ func TestEnsureBackendPoolDeletedConcurrently(t *testing.T) {
 		i := i
 		id := id
 		testFunc = append(testFunc, func() error {
-			_, err = ss.EnsureBackendPoolDeleted(&v1.Service{}, id, testVMSSNames[i], backendAddressPools, true)
+			_, err := ss.EnsureBackendPoolDeleted(&v1.Service{}, id, testVMSSNames[i], backendAddressPools, true)
 			return err
 		})
 	}