Repository: kubernetes-sigs/cloud-provider-azure
Pull request #2952 (2022-12-15 13:36:03)

[release-1.26] Use TimedCache.Get() for read-only resources
This is an automated cherry-pick of #2949

/assign lzhecheng

```release-note

```

k8s-ci-robot: @k8s-infra-cherrypick-robot: Adding the "do-not-merge/release-note-label-needed" label because no release-note block was detected, please follow our [release note process](https://git.k8s.io/community/contributors/guide/release-notes.md) to remove it.

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository.
</details>
lzhecheng: /kind bug

Patch:
diff --git a/pkg/cache/azure_cache.go b/pkg/cache/azure_cache.go
index b9e545bb7..d165c1f5f 100644
--- a/pkg/cache/azure_cache.go
+++ b/pkg/cache/azure_cache.go
@@ -122,8 +122,13 @@ func (t *TimedCache) getInternal(key string) (*AzureCacheEntry, error) {
 	return newEntry, nil
 }
 
-// Get returns the requested item by key with deep copy.
+// Get returns the requested item by key.
 func (t *TimedCache) Get(key string, crt AzureCacheReadType) (interface{}, error) {
+	return t.get(key, crt)
+}
+
+// Get returns the requested item by key with deep copy.
+func (t *TimedCache) GetWithDeepCopy(key string, crt AzureCacheReadType) (interface{}, error) {
 	data, err := t.get(key, crt)
 	copied := deepcopy.Copy(data)
 	return copied, err
diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go
index 8bf079129..03cd09d7f 100644
--- a/pkg/provider/azure_vmss.go
+++ b/pkg/provider/azure_vmss.go
@@ -81,11 +81,14 @@ type ScaleSet struct {
 	// the same cluster.
 	flexScaleSet VMSet
 
+	// vmssCache is timed cache where the Store in the cache is a map of
+	// Key: consts.VMSSKey
+	// Value: sync.Map of [vmssName]*VMSSEntry
 	vmssCache *azcache.TimedCache
 
 	// vmssVMCache is timed cache where the Store in the cache is a map of
 	// Key: [resourcegroup/vmssName]
-	// Value: sync.Map of [vmName]*VMSSVirtualMachinesEntry
+	// Value: sync.Map of [vmName]*VMSSVirtualMachineEntry
 	vmssVMCache *azcache.TimedCache
 
 	// nonVmssUniformNodesCache is used to store node names from non uniform vm.
@@ -195,7 +198,7 @@ func (ss *ScaleSet) getVmssVMByNodeIdentity(node *nodeIdentity, crt azcache.Azur
 		}
 
 		if entry, ok := virtualMachines.Load(node.nodeName); ok {
-			result := entry.(*VMSSVirtualMachinesEntry)
+			result := entry.(*VMSSVirtualMachineEntry)
 			if result.VirtualMachine == nil {
 				klog.Warningf("VM is nil on Node %q, VM is in deleting state", node.nodeName)
 				return nil, true, nil
@@ -331,7 +334,7 @@ func (ss *ScaleSet) getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceI
 		}
 
 		virtualMachines.Range(func(key, value interface{}) bool {
-			vmEntry := value.(*VMSSVirtualMachinesEntry)
+			vmEntry := value.(*VMSSVirtualMachineEntry)
 			if strings.EqualFold(vmEntry.ResourceGroup, resourceGroup) &&
 				strings.EqualFold(vmEntry.VMSSName, scaleSetName) &&
 				strings.EqualFold(vmEntry.InstanceID, instanceID) {
diff --git a/pkg/provider/azure_vmss_cache.go b/pkg/provider/azure_vmss_cache.go
index 903f9a157..47ff879cb 100644
--- a/pkg/provider/azure_vmss_cache.go
+++ b/pkg/provider/azure_vmss_cache.go
@@ -33,7 +33,7 @@ import (
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 )
 
-type VMSSVirtualMachinesEntry struct {
+type VMSSVirtualMachineEntry struct {
 	ResourceGroup  string
 	VMSSName       string
 	InstanceID     string
@@ -162,9 +162,9 @@ func (ss *ScaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) {
 	vmssVirtualMachinesCacheTTL := time.Duration(ss.Config.VmssVirtualMachinesCacheTTLInSeconds) * time.Second
 
 	getter := func(cacheKey string) (interface{}, error) {
-		localCache := &sync.Map{} // [nodeName]*VMSSVirtualMachinesEntry
+		localCache := &sync.Map{} // [nodeName]*VMSSVirtualMachineEntry
 
-		oldCache := make(map[string]*VMSSVirtualMachinesEntry)
+		oldCache := make(map[string]*VMSSVirtualMachineEntry)
 
 		entry, exists, err := ss.vmssVMCache.Store.GetByKey(cacheKey)
 		if err != nil {
@@ -175,7 +175,7 @@ func (ss *ScaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) {
 			if cached != nil {
 				virtualMachines := cached.(*sync.Map)
 				virtualMachines.Range(func(key, value interface{}) bool {
-					oldCache[key.(string)] = value.(*VMSSVirtualMachinesEntry)
+					oldCache[key.(string)] = value.(*VMSSVirtualMachineEntry)
 					return true
 				})
 			}
@@ -207,7 +207,7 @@ func (ss *ScaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) {
 				continue
 			}
 
-			vmssVMCacheEntry := &VMSSVirtualMachinesEntry{
+			vmssVMCacheEntry := &VMSSVirtualMachineEntry{
 				ResourceGroup:  resourceGroupName,
 				VMSSName:       vmssName,
 				InstanceID:     to.String(vm.InstanceID),
@@ -242,7 +242,7 @@ func (ss *ScaleSet) newVMSSVirtualMachinesCache() (*azcache.TimedCache, error) {
 			}
 
 			klog.V(5).Infof("adding old entries to new cache for %s", name)
-			localCache.Store(name, &VMSSVirtualMachinesEntry{
+			localCache.Store(name, &VMSSVirtualMachineEntry{
 				ResourceGroup:  vmEntry.ResourceGroup,
 				VMSSName:       vmEntry.VMSSName,
 				InstanceID:     vmEntry.InstanceID,
@@ -286,7 +286,6 @@ func (ss *ScaleSet) DeleteCacheForNode(nodeName string) error {
 		return err
 	}
 
-	// Delete in VMSS VM cache
 	if err := ss.gcVMSSVMCache(); err != nil {
 		klog.Errorf("DeleteCacheForNode(%s) failed to gc stale vmss caches: %v", nodeName, err)
 	}
@@ -308,7 +307,7 @@ func (ss *ScaleSet) updateCache(nodeName, resourceGroupName, vmssName, instanceI
 
 	localCache := &sync.Map{}
 
-	vmssVMCacheEntry := &VMSSVirtualMachinesEntry{
+	vmssVMCacheEntry := &VMSSVirtualMachineEntry{
 		ResourceGroup:  resourceGroupName,
 		VMSSName:       vmssName,
 		InstanceID:     instanceID,
@@ -320,7 +319,7 @@ func (ss *ScaleSet) updateCache(nodeName, resourceGroupName, vmssName, instanceI
 
 	virtualMachines.Range(func(key, value interface{}) bool {
 		if key.(string) != nodeName {
-			localCache.Store(key.(string), value.(*VMSSVirtualMachinesEntry))
+			localCache.Store(key.(string), value.(*VMSSVirtualMachineEntry))
 		}
 		return true
 	})
diff --git a/pkg/provider/azure_wrap.go b/pkg/provider/azure_wrap.go
index 71a86852d..05e965ea8 100644
--- a/pkg/provider/azure_wrap.go
+++ b/pkg/provider/azure_wrap.go
@@ -86,7 +86,7 @@ func (az *Cloud) getRouteTable(crt azcache.AzureCacheReadType) (routeTable netwo
 		return routeTable, false, fmt.Errorf("Route table name is not configured")
 	}
 
-	cachedRt, err := az.rtCache.Get(az.RouteTableName, crt)
+	cachedRt, err := az.rtCache.GetWithDeepCopy(az.RouteTableName, crt)
 	if err != nil {
 		return routeTable, false, err
 	}
@@ -109,7 +109,7 @@ func (az *Cloud) getPIPCacheKey(pipResourceGroup string, pipName string) string
 func (az *Cloud) getPublicIPAddress(pipResourceGroup string, pipName string, crt azcache.AzureCacheReadType) (network.PublicIPAddress, bool, error) {
 	pip := network.PublicIPAddress{}
 	cacheKey := az.getPIPCacheKey(pipResourceGroup, pipName)
-	cachedPIP, err := az.pipCache.Get(cacheKey, crt)
+	cachedPIP, err := az.pipCache.GetWithDeepCopy(cacheKey, crt)
 	if err != nil {
 		return pip, false, err
 	}
@@ -146,7 +146,7 @@ func (az *Cloud) getSubnet(virtualNetworkName string, subnetName string) (networ
 }
 
 func (az *Cloud) getAzureLoadBalancer(name string, crt azcache.AzureCacheReadType) (lb *network.LoadBalancer, exists bool, err error) {
-	cachedLB, err := az.lbCache.Get(name, crt)
+	cachedLB, err := az.lbCache.GetWithDeepCopy(name, crt)
 	if err != nil {
 		return lb, false, err
 	}
@@ -164,7 +164,7 @@ func (az *Cloud) getSecurityGroup(crt azcache.AzureCacheReadType) (network.Secur
 		return nsg, fmt.Errorf("securityGroupName is not configured")
 	}
 
-	securityGroup, err := az.nsgCache.Get(az.SecurityGroupName, crt)
+	securityGroup, err := az.nsgCache.GetWithDeepCopy(az.SecurityGroupName, crt)
 	if err != nil {
 		return nsg, err
 	}
@@ -177,7 +177,7 @@ func (az *Cloud) getSecurityGroup(crt azcache.AzureCacheReadType) (network.Secur
 }
 
 func (az *Cloud) getPrivateLinkService(frontendIPConfigID *string, crt azcache.AzureCacheReadType) (pls network.PrivateLinkService, err error) {
-	cachedPLS, err := az.plsCache.Get(*frontendIPConfigID, crt)
+	cachedPLS, err := az.plsCache.GetWithDeepCopy(*frontendIPConfigID, crt)
 	if err != nil {
 		return pls, err
 	}


Test patch:
diff --git a/pkg/cache/azure_cache_test.go b/pkg/cache/azure_cache_test.go
index 50b110d59..82e30d0e6 100644
--- a/pkg/cache/azure_cache_test.go
+++ b/pkg/cache/azure_cache_test.go
@@ -98,7 +98,7 @@ func TestCacheGet(t *testing.T) {
 	for _, c := range cases {
 		dataSource, cache := newFakeCache(t)
 		dataSource.set(c.data)
-		val, err := cache.Get(c.key, CacheReadTypeDefault)
+		val, err := cache.GetWithDeepCopy(c.key, CacheReadTypeDefault)
 		assert.NoError(t, err, c.name)
 		assert.Equal(t, c.expected, val, c.name)
 	}
@@ -112,7 +112,7 @@ func TestCacheGetError(t *testing.T) {
 	cache, err := NewTimedcache(fakeCacheTTL, getter)
 	assert.NoError(t, err)
 
-	val, err := cache.Get("key", CacheReadTypeDefault)
+	val, err := cache.GetWithDeepCopy("key", CacheReadTypeDefault)
 	assert.Error(t, err)
 	assert.Equal(t, getError, err)
 	assert.Nil(t, val)
@@ -140,7 +140,7 @@ func TestCacheGetWithDeepCopy(t *testing.T) {
 			dataSource, cache := newFakeCache(t)
 			dataSource.set(c.data)
 			cache.Set(c.key, valFake)
-			val, err := cache.Get(c.key, CacheReadTypeDefault)
+			val, err := cache.GetWithDeepCopy(c.key, CacheReadTypeDefault)
 			assert.NoError(t, err)
 			assert.Equal(t, c.expected, val.(*fakeDataObj).Data)
 
@@ -160,13 +160,13 @@ func TestCacheDelete(t *testing.T) {
 	dataSource, cache := newFakeCache(t)
 	dataSource.set(data)
 
-	v, err := cache.Get(testKey, CacheReadTypeDefault)
+	v, err := cache.GetWithDeepCopy(testKey, CacheReadTypeDefault)
 	assert.NoError(t, err)
 	assert.Equal(t, val, v, "cache should get correct data")
 
 	dataSource.set(nil)
 	_ = cache.Delete(testKey)
-	v, err = cache.Get(testKey, CacheReadTypeDefault)
+	v, err = cache.GetWithDeepCopy(testKey, CacheReadTypeDefault)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, dataSource.called)
 	assert.Equal(t, nil, v, "cache should get nil after data is removed")
@@ -180,13 +180,13 @@ func TestCacheExpired(t *testing.T) {
 	dataSource, cache := newFakeCache(t)
 	dataSource.set(data)
 
-	v, err := cache.Get(testKey, CacheReadTypeDefault)
+	v, err := cache.GetWithDeepCopy(testKey, CacheReadTypeDefault)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, dataSource.called)
 	assert.Equal(t, val, v, "cache should get correct data")
 
 	time.Sleep(fakeCacheTTL)
-	v, err = cache.Get(testKey, CacheReadTypeDefault)
+	v, err = cache.GetWithDeepCopy(testKey, CacheReadTypeDefault)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, dataSource.called)
 	assert.Equal(t, val, v, "cache should get correct data even after expired")
@@ -200,13 +200,13 @@ func TestCacheAllowUnsafeRead(t *testing.T) {
 	dataSource, cache := newFakeCache(t)
 	dataSource.set(data)
 
-	v, err := cache.Get(testKey, CacheReadTypeDefault)
+	v, err := cache.GetWithDeepCopy(testKey, CacheReadTypeDefault)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, dataSource.called)
 	assert.Equal(t, val, v, "cache should get correct data")
 
 	time.Sleep(fakeCacheTTL)
-	v, err = cache.Get(testKey, CacheReadTypeUnsafe)
+	v, err = cache.GetWithDeepCopy(testKey, CacheReadTypeUnsafe)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, dataSource.called)
 	assert.Equal(t, val, v, "cache should return expired as allow unsafe read is allowed")
@@ -226,10 +226,10 @@ func TestCacheNoConcurrentGet(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			_, _ = cache.Get(testKey, CacheReadTypeDefault)
+			_, _ = cache.GetWithDeepCopy(testKey, CacheReadTypeDefault)
 		}()
 	}
-	v, err := cache.Get(testKey, CacheReadTypeDefault)
+	v, err := cache.GetWithDeepCopy(testKey, CacheReadTypeDefault)
 	wg.Wait()
 	assert.NoError(t, err)
 	assert.Equal(t, 1, dataSource.called)
@@ -244,12 +244,12 @@ func TestCacheForceRefresh(t *testing.T) {
 	dataSource, cache := newFakeCache(t)
 	dataSource.set(data)
 
-	v, err := cache.Get(testKey, CacheReadTypeDefault)
+	v, err := cache.GetWithDeepCopy(testKey, CacheReadTypeDefault)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, dataSource.called)
 	assert.Equal(t, val, v, "cache should get correct data")
 
-	v, err = cache.Get(testKey, CacheReadTypeForceRefresh)
+	v, err = cache.GetWithDeepCopy(testKey, CacheReadTypeForceRefresh)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, dataSource.called)
 	assert.Equal(t, val, v, "should refetch unexpired data as forced refresh")
diff --git a/pkg/provider/azure_backoff_test.go b/pkg/provider/azure_backoff_test.go
index a661b05b0..76db3e1b7 100644
--- a/pkg/provider/azure_backoff_test.go
+++ b/pkg/provider/azure_backoff_test.go
@@ -234,7 +234,7 @@ func TestCreateOrUpdateSecurityGroupCanceled(t *testing.T) {
 	assert.EqualError(t, fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: %w", fmt.Errorf("canceledandsupersededduetoanotheroperation")), err.Error())
 
 	// security group should be removed from cache if the operation is canceled
-	shouldBeEmpty, err := az.nsgCache.Get("sg", cache.CacheReadTypeDefault)
+	shouldBeEmpty, err := az.nsgCache.GetWithDeepCopy("sg", cache.CacheReadTypeDefault)
 	assert.NoError(t, err)
 	assert.Empty(t, shouldBeEmpty)
 }
@@ -287,12 +287,12 @@ func TestCreateOrUpdateLB(t *testing.T) {
 		assert.EqualError(t, test.expectedErr, err.Error())
 
 		// loadbalancer should be removed from cache if the etag is mismatch or the operation is canceled
-		shouldBeEmpty, err := az.lbCache.Get("lb", cache.CacheReadTypeDefault)
+		shouldBeEmpty, err := az.lbCache.GetWithDeepCopy("lb", cache.CacheReadTypeDefault)
 		assert.NoError(t, err)
 		assert.Empty(t, shouldBeEmpty)
 
 		// public ip cache should be populated since there's GetPIP
-		shouldNotBeEmpty, err := az.pipCache.Get(az.getPIPCacheKey(az.ResourceGroup, "pip"), cache.CacheReadTypeDefault)
+		shouldNotBeEmpty, err := az.pipCache.GetWithDeepCopy(az.getPIPCacheKey(az.ResourceGroup, "pip"), cache.CacheReadTypeDefault)
 		assert.NoError(t, err)
 		assert.NotEmpty(t, shouldNotBeEmpty)
 	}
@@ -417,7 +417,7 @@ func TestCreateOrUpdatePIP(t *testing.T) {
 		err := az.CreateOrUpdatePIP(&v1.Service{}, az.ResourceGroup, network.PublicIPAddress{Name: to.StringPtr("nic")})
 		assert.EqualError(t, test.expectedErr, err.Error())
 
-		cachedPIP, err := az.pipCache.Get(az.getPIPCacheKey(az.ResourceGroup, "nic"), cache.CacheReadTypeDefault)
+		cachedPIP, err := az.pipCache.GetWithDeepCopy(az.getPIPCacheKey(az.ResourceGroup, "nic"), cache.CacheReadTypeDefault)
 		assert.NoError(t, err)
 		if test.cacheExpectedEmpty {
 			assert.Empty(t, cachedPIP)
@@ -496,7 +496,7 @@ func TestCreateOrUpdateRouteTable(t *testing.T) {
 		assert.EqualError(t, test.expectedErr, err.Error())
 
 		// route table should be removed from cache if the etag is mismatch or the operation is canceled
-		shouldBeEmpty, err := az.rtCache.Get("rt", cache.CacheReadTypeDefault)
+		shouldBeEmpty, err := az.rtCache.GetWithDeepCopy("rt", cache.CacheReadTypeDefault)
 		assert.NoError(t, err)
 		assert.Empty(t, shouldBeEmpty)
 	}
@@ -542,7 +542,7 @@ func TestCreateOrUpdateRoute(t *testing.T) {
 			assert.EqualError(t, test.expectedErr, err.Error())
 		}
 
-		shouldBeEmpty, err := az.rtCache.Get("rt", cache.CacheReadTypeDefault)
+		shouldBeEmpty, err := az.rtCache.GetWithDeepCopy("rt", cache.CacheReadTypeDefault)
 		assert.NoError(t, err)
 		assert.Empty(t, shouldBeEmpty)
 	}
diff --git a/pkg/provider/azure_vmss_test.go b/pkg/provider/azure_vmss_test.go
index c5fb0e83b..aa1a29832 100644
--- a/pkg/provider/azure_vmss_test.go
+++ b/pkg/provider/azure_vmss_test.go
@@ -959,7 +959,7 @@ func TestGetVmssVMByNodeIdentity(t *testing.T) {
 			virtualMachines, err := ss.getVMSSVMsFromCache(ss.ResourceGroup, testVMSSName, azcache.CacheReadTypeDefault)
 			assert.Nil(t, err)
 			for _, vm := range test.goneVMList {
-				entry := VMSSVirtualMachinesEntry{
+				entry := VMSSVirtualMachineEntry{
 					ResourceGroup: ss.ResourceGroup,
 					VMSSName:      testVMSSName,
 				}
diff --git a/tests/e2e/network/node.go b/tests/e2e/network/node.go
index 0460246bc..a0e7f5ede 100644
--- a/tests/e2e/network/node.go
+++ b/tests/e2e/network/node.go
@@ -322,11 +322,11 @@ var _ = Describe("Azure nodes", func() {
 
 		utils.Logf("scaling VMSS")
 		count := *vmss.Sku.Capacity
-		err = utils.ScaleVMSS(tc, *vmss.Name, tc.GetResourceGroup(), int64(vmssScaleUpCelling))
+		err = utils.ScaleVMSS(tc, *vmss.Name, int64(vmssScaleUpCelling))
 
 		defer func() {
 			utils.Logf("restoring VMSS")
-			err = utils.ScaleVMSS(tc, *vmss.Name, tc.GetResourceGroup(), count)
+			err = utils.ScaleVMSS(tc, *vmss.Name, count)
 			Expect(err).NotTo(HaveOccurred())
 		}()
 
diff --git a/tests/e2e/node/vmss.go b/tests/e2e/node/vmss.go
index 7f11e9a1a..07094fd67 100644
--- a/tests/e2e/node/vmss.go
+++ b/tests/e2e/node/vmss.go
@@ -17,9 +17,6 @@ limitations under the License.
 package node
 
 import (
-	"os"
-	"strings"
-
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -30,7 +27,7 @@ import (
 	"sigs.k8s.io/cloud-provider-azure/tests/e2e/utils"
 )
 
-var _ = Describe("Lifecycle of VMSS", Label(utils.TestSuiteLabelVMSS), func() {
+var _ = Describe("Lifecycle of VMSS", Label(utils.TestSuiteLabelVMSS, utils.TestSuiteLabelVMSSScale), func() {
 	var (
 		ns     *v1.Namespace
 		k8sCli kubernetes.Interface
@@ -73,21 +70,13 @@ var _ = Describe("Lifecycle of VMSS", Label(utils.TestSuiteLabelVMSS), func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("deallocate VMSS instance")
-		if strings.EqualFold(os.Getenv(utils.CAPZTestCCM), "true") {
-			err = utils.ScaleMachinePool(*vmss.Name, numInstance-1)
-		} else {
-			err = utils.ScaleVMSS(azCli, *vmss.Name, azCli.GetResourceGroup(), numInstance-1)
-		}
+		err = utils.Scale(azCli, *vmss.Name, numInstance-1)
 		Expect(err).NotTo(HaveOccurred())
 		expectedCap[*vmss.Name] = numInstance - 1
 
 		defer func() {
 			By("reset VMSS instance")
-			if strings.EqualFold(os.Getenv(utils.CAPZTestCCM), "true") {
-				err = utils.ScaleMachinePool(*vmss.Name, numInstance)
-			} else {
-				err = utils.ScaleVMSS(azCli, *vmss.Name, azCli.GetResourceGroup(), numInstance)
-			}
+			err = utils.Scale(azCli, *vmss.Name, numInstance)
 			Expect(err).NotTo(HaveOccurred())
 			expectedCap[*vmss.Name] = numInstance
 
@@ -117,21 +106,13 @@ var _ = Describe("Lifecycle of VMSS", Label(utils.TestSuiteLabelVMSS), func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("allocate VMSS instance")
-		if strings.EqualFold(os.Getenv(utils.CAPZTestCCM), "true") {
-			err = utils.ScaleMachinePool(*vmss.Name, numInstance+1)
-		} else {
-			err = utils.ScaleVMSS(azCli, *vmss.Name, azCli.GetResourceGroup(), numInstance+1)
-		}
+		err = utils.Scale(azCli, *vmss.Name, numInstance+1)
 		Expect(err).NotTo(HaveOccurred())
 		expectedCap[*vmss.Name] = numInstance + 1
 
 		defer func() {
 			By("reset VMSS instance")
-			if strings.EqualFold(os.Getenv(utils.CAPZTestCCM), "true") {
-				err = utils.ScaleMachinePool(*vmss.Name, numInstance)
-			} else {
-				err = utils.ScaleVMSS(azCli, *vmss.Name, azCli.GetResourceGroup(), numInstance)
-			}
+			err = utils.Scale(azCli, *vmss.Name, numInstance)
 			Expect(err).NotTo(HaveOccurred())
 			expectedCap[*vmss.Name] = numInstance
 
diff --git a/tests/e2e/utils/consts.go b/tests/e2e/utils/consts.go
index ab9b60985..f85a125fe 100644
--- a/tests/e2e/utils/consts.go
+++ b/tests/e2e/utils/consts.go
@@ -24,6 +24,7 @@ const (
 	TestSuiteLabelMultiNodePools     = "Multi-Nodepool"
 	TestSuiteLabelSingleNodePool     = "Single-Nodepool"
 	TestSuiteLabelVMSS               = "VMSS"
+	TestSuiteLabelVMSSScale          = "VMSS-Scale"
 	TestSuiteLabelSpotVM             = "Spot-VM"
 	TestSuiteLabelKubenet            = "Kubenet"
 	TestSuiteLabelMultiGroup         = "Multi-Group"
diff --git a/tests/e2e/utils/vmss_utils.go b/tests/e2e/utils/vmss_utils.go
index 08350ffea..4a4e74c4e 100644
--- a/tests/e2e/utils/vmss_utils.go
+++ b/tests/e2e/utils/vmss_utils.go
@@ -19,6 +19,7 @@ package utils
 import (
 	"context"
 	"fmt"
+	"os"
 	"regexp"
 	"strings"
 
@@ -53,9 +54,17 @@ func FindTestVMSS(tc *AzureTestClient, rgName string) (*azcompute.VirtualMachine
 	return &vmssList[0], nil
 }
 
+func Scale(tc *AzureTestClient, vmssName string, instanceCount int64) error {
+	if strings.EqualFold(os.Getenv(CAPZTestCCM), "true") {
+		return ScaleMachinePool(vmssName, instanceCount)
+	}
+	return ScaleVMSS(tc, vmssName, instanceCount)
+}
+
 // ScaleVMSS scales the given VMSS
-func ScaleVMSS(tc *AzureTestClient, vmssName, rgName string, instanceCount int64) (err error) {
+func ScaleVMSS(tc *AzureTestClient, vmssName string, instanceCount int64) (err error) {
 	Logf("ScaleVMSS: start")
+	rgName := tc.GetResourceGroup()
 
 	vmssClient := tc.createVMSSClient()
 
@@ -178,10 +187,11 @@ func ValidateClusterNodesMatchVMSSInstances(tc *AzureTestClient, expectedCap map
 				Logf("Failed to validate VMSS instances: %s", err)
 			}
 
-			Logf("Matching cluster nodes[%s] with VMSS instances[%s]",
+			Logf("Matching cluster nodes[%s] with VMSS instances[%s]\nExpected capacity: %v, actual capacity: %v",
 				strings.Join(nodeSet.List(), ","),
-				strings.Join(instanceSet.List(), ","))
-			Logf("Expected capacity: %v, actual one: %v", expectedCap, actualCap)
+				strings.Join(instanceSet.List(), ","),
+				expectedCap,
+				actualCap)
 		}()
 
 		nodes, err = GetAgentNodes(k8sCli)
@@ -196,6 +206,10 @@ func ValidateClusterNodesMatchVMSSInstances(tc *AzureTestClient, expectedCap map
 		vmssList, _ := ListUniformVMSSes(tc)
 		capMatch := true
 		for _, vmss := range vmssList {
+			cap, ok := expectedCap[*vmss.Name]
+			if !ok {
+				continue
+			}
 			vms, err := ListVMSSVMs(tc, *vmss.Name)
 			if err != nil {
 				return false, err
@@ -210,15 +224,23 @@ func ValidateClusterNodesMatchVMSSInstances(tc *AzureTestClient, expectedCap map
 				vmssInstanceSet.Insert(strings.ToLower(nodeName))
 				instanceSet.Insert(strings.ToLower(nodeName))
 			}
-			cap, ok := expectedCap[*vmss.Name]
-			if ok {
-				actualCap[*vmss.Name] = *vmss.Sku.Capacity
-				// For autoscaling cluster, simply comparing the capacity may not work since if the number of current nodes is lower than the "minCount", a new node may be created after scaling down.
-				// In this situation, we compare the expected capacity with the length of intersection between original nodes and current nodes.
-				if cap != *vmss.Sku.Capacity && cap != int64(originalNodeSet.Intersection(vmssInstanceSet).Len()) {
+
+			actualCap[*vmss.Name] = *vmss.Sku.Capacity
+			if cap != *vmss.Sku.Capacity {
+				if !strings.Contains(os.Getenv(AKSClusterType), "autoscaling") {
 					capMatch = false
+					break
+				}
+				if cap != int64(originalNodeSet.Intersection(vmssInstanceSet).Len()) {
+					// For autoscaling cluster, simply comparing the capacity may not work since if the number of current nodes is lower than the "minCount", a new node may be created after scaling down.
+					// In this situation, we compare the expected capacity with the length of intersection between original nodes and current nodes.
 					Logf("VMSS %q sku capacity is expected to be %d, but actually %d", *vmss.Name, cap, *vmss.Sku.Capacity)
+					capMatch = false
+					break
 				}
+			} else if int64(len(nodeSet)) != cap {
+				capMatch = false
+				break
 			}
 		}