Repository: kubernetes-sigs/cloud-provider-azure
Pull request #5363 (2024-02-04 02:24:32)

[release-1.27] fix: Skip attaching/detaching vmss vm to lb backend pool if the vm is…
… not active.

We should not update the VM instance if its provisioning state or power state is not good. This will save a lot of api calls and reduce throttling issues.

<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?

<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

/kind bug

#### What this PR does / why we need it:

fix: Skip attaching/detaching vmss vm to lb backend pool if the vm is not active.

We should not update the VM instance if its provisioning state or power state is not good. This will save a lot of api calls and reduce throttling issues.

#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
fix: Skip attaching/detaching vmss vm to lb backend pool if the vm is not active.

We should not update the VM instance if its provisioning state or power state is not good. This will save a lot of api calls and reduce throttling issues.
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```



Patch:
diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go
index a755b47ee..e716b377d 100644
--- a/pkg/consts/consts.go
+++ b/pkg/consts/consts.go
@@ -113,8 +113,8 @@ const (
 	// StrRawVersion is the raw version string
 	StrRawVersion string = "raw"
 
-	// VirtualMachineScaleSetsDeallocating indicates VMSS instances are in Deallocating state.
-	VirtualMachineScaleSetsDeallocating = "Deallocating"
+	// ProvisionStateDeleting indicates VMSS instances are in Deleting state.
+	ProvisionStateDeleting = "Deleting"
 	// VmssMachineIDTemplate is the vmss manchine ID template
 	VmssMachineIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%s"
 	// VMSetCIDRIPV4TagKey specifies the node ipv4 CIDR mask of the instances on the VMSS or VMAS
@@ -130,6 +130,12 @@ const (
 	VMSetNamesSharingPrimarySLBDelimiter = ","
 	// PremiumV2_LRS type for Azure Disk
 	PremiumV2LRS = compute.DiskStorageAccountTypes("PremiumV2_LRS")
+	// ProvisioningStateDeleting ...
+	ProvisioningStateDeleting = "Deleting"
+	// ProvisioningStateSucceeded ...
+	ProvisioningStateSucceeded = "Succeeded"
+	// ProvisioningStateUnknown is the unknown provisioning state
+	ProvisioningStateUnknown = "Unknown"
 )
 
 // cache
@@ -543,3 +549,13 @@ const (
 const (
 	VMSSTagForBatchOperation = "aks-managed-coordination"
 )
+
+// VM power state
+const (
+	VMPowerStatePrefix       = "PowerState/"
+	VMPowerStateStopped      = "stopped"
+	VMPowerStateStopping     = "stopping"
+	VMPowerStateDeallocated  = "deallocated"
+	VMPowerStateDeallocating = "deallocating"
+	VMPowerStateUnknown      = "unknown"
+)
diff --git a/pkg/provider/azure_backoff.go b/pkg/provider/azure_backoff.go
index e5bf0db87..c672ce574 100644
--- a/pkg/provider/azure_backoff.go
+++ b/pkg/provider/azure_backoff.go
@@ -557,7 +557,7 @@ func (az *Cloud) CreateOrUpdateVMSS(resourceGroupName string, VMScaleSetName str
 		klog.Errorf("CreateOrUpdateVMSS: error getting vmss(%s): %v", VMScaleSetName, rerr)
 		return rerr
 	}
-	if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+	if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 		klog.V(3).Infof("CreateOrUpdateVMSS: found vmss %s being deleted, skipping", VMScaleSetName)
 		return nil
 	}
diff --git a/pkg/provider/azure_instances.go b/pkg/provider/azure_instances.go
index 55e7ea3f2..353cc289f 100644
--- a/pkg/provider/azure_instances.go
+++ b/pkg/provider/azure_instances.go
@@ -35,12 +35,6 @@ import (
 )
 
 const (
-	vmPowerStatePrefix       = "PowerState/"
-	vmPowerStateStopped      = "stopped"
-	vmPowerStateDeallocated  = "deallocated"
-	vmPowerStateDeallocating = "deallocating"
-	vmPowerStateUnknown      = "unknown"
-
 	// nodeNameEnvironmentName is the environment variable name for getting node name.
 	// It is only used for out-of-tree cloud provider.
 	nodeNameEnvironmentName = "NODE_NAME"
@@ -298,7 +292,7 @@ func (az *Cloud) InstanceShutdownByProviderID(ctx context.Context, providerID st
 
 	status := strings.ToLower(powerStatus)
 	provisioningSucceeded := strings.EqualFold(strings.ToLower(provisioningState), strings.ToLower(string(compute.ProvisioningStateSucceeded)))
-	return provisioningSucceeded && (status == vmPowerStateStopped || status == vmPowerStateDeallocated || status == vmPowerStateDeallocating), nil
+	return provisioningSucceeded && (status == consts.VMPowerStateStopped || status == consts.VMPowerStateDeallocated || status == consts.VMPowerStateDeallocating), nil
 }
 
 // InstanceShutdown returns true if the instance is shutdown according to the cloud provider.
diff --git a/pkg/provider/azure_standard.go b/pkg/provider/azure_standard.go
index 7aed1a429..de28b5233 100644
--- a/pkg/provider/azure_standard.go
+++ b/pkg/provider/azure_standard.go
@@ -45,6 +45,7 @@ import (
 
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
+	vmutil "sigs.k8s.io/cloud-provider-azure/pkg/util/vm"
 )
 
 var (
@@ -644,19 +645,13 @@ func (as *availabilitySet) GetPowerStatusByNodeName(name string) (powerState str
 		return powerState, err
 	}
 
-	if vm.InstanceView != nil && vm.InstanceView.Statuses != nil {
-		statuses := *vm.InstanceView.Statuses
-		for _, status := range statuses {
-			state := pointer.StringDeref(status.Code, "")
-			if strings.HasPrefix(state, vmPowerStatePrefix) {
-				return strings.TrimPrefix(state, vmPowerStatePrefix), nil
-			}
-		}
+	if vm.InstanceView != nil {
+		return vmutil.GetVMPowerState(pointer.StringDeref(vm.Name, ""), vm.InstanceView.Statuses), nil
 	}
 
 	// vm.InstanceView or vm.InstanceView.Statuses are nil when the VM is under deleting.
 	klog.V(3).Infof("InstanceView for node %q is nil, assuming it's deleting", name)
-	return vmPowerStateUnknown, nil
+	return consts.VMPowerStateUnknown, nil
 }
 
 // GetProvisioningStateByNodeName returns the provisioningState for the specified node.
diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go
index b2c116d2c..842d8fb8d 100644
--- a/pkg/provider/azure_vmss.go
+++ b/pkg/provider/azure_vmss.go
@@ -40,6 +40,7 @@ import (
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
 	"sigs.k8s.io/cloud-provider-azure/pkg/provider/virtualmachine"
+	vmutil "sigs.k8s.io/cloud-provider-azure/pkg/util/vm"
 )
 
 var (
@@ -281,20 +282,14 @@ func (ss *ScaleSet) GetPowerStatusByNodeName(name string) (powerState string, er
 
 	if vm.IsVirtualMachineScaleSetVM() {
 		v := vm.AsVirtualMachineScaleSetVM()
-		if v.InstanceView != nil && v.InstanceView.Statuses != nil {
-			statuses := *v.InstanceView.Statuses
-			for _, status := range statuses {
-				state := pointer.StringDeref(status.Code, "")
-				if strings.HasPrefix(state, vmPowerStatePrefix) {
-					return strings.TrimPrefix(state, vmPowerStatePrefix), nil
-				}
-			}
+		if v.InstanceView != nil {
+			return vmutil.GetVMPowerState(pointer.StringDeref(v.Name, ""), v.InstanceView.Statuses), nil
 		}
 	}
 
 	// vm.InstanceView or vm.InstanceView.Statuses are nil when the VM is under deleting.
 	klog.V(3).Infof("InstanceView for node %q is nil, assuming it's deleting", name)
-	return vmPowerStateUnknown, nil
+	return consts.VMPowerStateUnknown, nil
 }
 
 // GetProvisioningStateByNodeName returns the provisioningState for the specified node.
@@ -1069,6 +1064,8 @@ func getPrimaryIPConfigFromVMSSNetworkConfig(config *compute.VirtualMachineScale
 // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
 // participating in the specified LoadBalancer Backend Pool, which returns (resourceGroup, vmasName, instanceID, vmssVM, error).
 func (ss *ScaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetNameOfLB string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
+	logger := klog.Background().WithName("EnsureHostInPool").
+		WithValues("nodeName", nodeName, "backendPoolID", backendPoolID, "vmSetNameOfLB", vmSetNameOfLB)
 	vmName := mapNodeNameToVMName(nodeName)
 	vm, err := ss.getVmssVM(vmName, azcache.CacheReadTypeDefault)
 	if err != nil {
@@ -1077,13 +1074,20 @@ func (ss *ScaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam
 			return "", "", "", nil, nil
 		}
 
-		klog.Errorf("EnsureHostInPool: failed to get VMSS VM %s: %v", vmName, err)
+		logger.Error(err, "failed to get vmss vm", "vmName", vmName)
 		if !errors.Is(err, ErrorNotVmssInstance) {
 			return "", "", "", nil, err
 		}
 	}
+	statuses := vm.GetInstanceViewStatus()
+	vmPowerState := vmutil.GetVMPowerState(vm.Name, statuses)
+	provisioningState := vm.GetProvisioningState()
+	if vmutil.IsNotActiveVMState(provisioningState, vmPowerState) {
+		logger.V(2).Info("skip updating the node because it is not in an active state", "vmName", vmName, "provisioningState", provisioningState, "vmPowerState", vmPowerState)
+		return "", "", "", nil, nil
+	}
 
-	klog.V(2).Infof("ensuring node %q of scaleset %q in LB backendpool %q", nodeName, vm.VMSSName, backendPoolID)
+	logger.V(2).Info("ensuring the vmss node in LB backendpool", "vmss name", vm.VMSSName)
 
 	// Check scale set name:
 	// - For basic SKU load balancer, return nil if the node's scale set is mismatched with vmSetNameOfLB.
@@ -1108,14 +1112,13 @@ func (ss *ScaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam
 	}
 
 	if vmSetNameOfLB != "" && needCheck && !strings.EqualFold(vmSetNameOfLB, vm.VMSSName) {
-		klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the ScaleSet %s", vmName, vmSetNameOfLB)
+		logger.V(3).Info("skips the node %s because it is not in the ScaleSet %s", vmName, vmSetNameOfLB)
 		return "", "", "", nil, nil
 	}
 
 	// Find primary network interface configuration.
 	if vm.VirtualMachineScaleSetVMProperties.NetworkProfileConfiguration.NetworkInterfaceConfigurations == nil {
-		klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vm %s, "+
-			"probably because the vm's being deleted", vmName)
+		logger.V(4).Info("cannot obtain the primary network interface configuration, of the vm, probably because the vm's being deleted", "vmName", vmName)
 		return "", "", "", nil, nil
 	}
 
@@ -1165,7 +1168,7 @@ func (ss *ScaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam
 			return "", "", "", nil, err
 		}
 		if !isSameLB {
-			klog.V(4).Infof("Node %q has already been added to LB %q, omit adding it to a new one", nodeName, oldLBName)
+			logger.V(4).Info("The node has already been added to an LB, omit adding it to a new one", "lbName", oldLBName)
 			return "", "", "", nil, nil
 		}
 	}
@@ -1272,7 +1275,7 @@ func (ss *ScaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 			klog.V(3).Infof("ensureVMSSInPool: found vmss %s being deleted, skipping", vmssName)
 			continue
 		}
@@ -1544,6 +1547,7 @@ func (ss *ScaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac
 // ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted
 // from the specified node, which returns (resourceGroup, vmasName, instanceID, vmssVM, error).
 func (ss *ScaleSet) ensureBackendPoolDeletedFromNode(nodeName string, backendPoolIDs []string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
+	logger := klog.Background().WithName("ensureBackendPoolDeletedFromNode").WithValues("nodeName", nodeName, "backendPoolIDs", backendPoolIDs)
 	vm, err := ss.getVmssVM(nodeName, azcache.CacheReadTypeDefault)
 	if err != nil {
 		if errors.Is(err, cloudprovider.InstanceNotFound) {
@@ -1554,6 +1558,14 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromNode(nodeName string, backendPoo
 		return "", "", "", nil, err
 	}
 
+	statuses := vm.GetInstanceViewStatus()
+	vmPowerState := vmutil.GetVMPowerState(vm.Name, statuses)
+	provisioningState := vm.GetProvisioningState()
+	if vmutil.IsNotActiveVMState(provisioningState, vmPowerState) {
+		logger.V(2).Info("skip updating the node because it is not in an active state", "provisioningState", provisioningState, "vmPowerState", vmPowerState)
+		return "", "", "", nil, nil
+	}
+
 	// Find primary network interface configuration.
 	if vm.VirtualMachineScaleSetVMProperties.NetworkProfileConfiguration.NetworkInterfaceConfigurations == nil {
 		klog.V(4).Infof("ensureBackendPoolDeletedFromNode: cannot obtain the primary network interface configuration, of vm %s, "+
@@ -1730,7 +1742,7 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromVmssUniform(backendPoolIDs []str
 
 			// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 			// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-			if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+			if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 				klog.V(3).Infof("ensureBackendPoolDeletedFromVMSS: found vmss %s being deleted, skipping", pointer.StringDeref(vmss.Name, ""))
 				return true
 			}
@@ -2137,7 +2149,7 @@ func (ss *ScaleSet) EnsureBackendPoolDeletedFromVMSets(vmssNamesMap map[string]b
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 			klog.V(3).Infof("EnsureBackendPoolDeletedFromVMSets: found vmss %s being deleted, skipping", vmssName)
 			continue
 		}
diff --git a/pkg/provider/azure_vmssflex.go b/pkg/provider/azure_vmssflex.go
index c79aa7c5b..fdd5a9567 100644
--- a/pkg/provider/azure_vmssflex.go
+++ b/pkg/provider/azure_vmssflex.go
@@ -27,6 +27,7 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
+
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -37,6 +38,7 @@ import (
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
+	vmutil "sigs.k8s.io/cloud-provider-azure/pkg/util/vm"
 )
 
 var (
@@ -269,19 +271,13 @@ func (fs *FlexScaleSet) GetPowerStatusByNodeName(name string) (powerState string
 		return powerState, err
 	}
 
-	if vm.InstanceView != nil && vm.InstanceView.Statuses != nil {
-		statuses := *vm.InstanceView.Statuses
-		for _, status := range statuses {
-			state := pointer.StringDeref(status.Code, "")
-			if strings.HasPrefix(state, vmPowerStatePrefix) {
-				return strings.TrimPrefix(state, vmPowerStatePrefix), nil
-			}
-		}
+	if vm.InstanceView != nil {
+		return vmutil.GetVMPowerState(pointer.StringDeref(vm.Name, ""), vm.InstanceView.Statuses), nil
 	}
 
 	// vm.InstanceView or vm.InstanceView.Statuses are nil when the VM is under deleting.
 	klog.V(3).Infof("InstanceView for node %q is nil, assuming it's deleting", name)
-	return vmPowerStateUnknown, nil
+	return consts.VMPowerStateUnknown, nil
 }
 
 // GetPrimaryInterface gets machine primary network interface by node name.
@@ -630,7 +626,7 @@ func (fs *FlexScaleSet) ensureVMSSFlexInPool(service *v1.Service, nodes []*v1.No
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmssFlex.ProvisioningState != nil && strings.EqualFold(*vmssFlex.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmssFlex.ProvisioningState != nil && strings.EqualFold(*vmssFlex.ProvisioningState, consts.ProvisionStateDeleting) {
 			klog.V(3).Infof("ensureVMSSFlexInPool: found vmss %s being deleted, skipping", vmssFlexID)
 			continue
 		}
@@ -804,7 +800,7 @@ func (fs *FlexScaleSet) EnsureBackendPoolDeletedFromVMSets(vmssNamesMap map[stri
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 			klog.V(3).Infof("fs.EnsureBackendPoolDeletedFromVMSets: found vmss %s being deleted, skipping", vmssName)
 			continue
 		}
diff --git a/pkg/provider/virtualmachine/virtualmachine.go b/pkg/provider/virtualmachine/virtualmachine.go
index 7e37c7036..2d937c6b7 100644
--- a/pkg/provider/virtualmachine/virtualmachine.go
+++ b/pkg/provider/virtualmachine/virtualmachine.go
@@ -19,6 +19,8 @@ package virtualmachine
 import (
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute"
 	"k8s.io/utils/pointer"
+
+	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 )
 
 type Variant string
@@ -145,6 +147,36 @@ func (vm *VirtualMachine) AsVirtualMachineScaleSetVM() *compute.VirtualMachineSc
 	return vm.vmssVM
 }
 
+func (vm *VirtualMachine) GetInstanceViewStatus() *[]compute.InstanceViewStatus {
+	if vm.IsVirtualMachine() && vm.vm != nil &&
+		vm.vm.VirtualMachineProperties != nil &&
+		vm.vm.VirtualMachineProperties.InstanceView != nil {
+		return vm.vm.VirtualMachineProperties.InstanceView.Statuses
+	}
+	if vm.IsVirtualMachineScaleSetVM() &&
+		vm.vmssVM != nil &&
+		vm.vmssVM.VirtualMachineScaleSetVMProperties != nil &&
+		vm.vmssVM.VirtualMachineScaleSetVMProperties.InstanceView != nil {
+		return vm.vmssVM.VirtualMachineScaleSetVMProperties.InstanceView.Statuses
+	}
+	return nil
+}
+
+func (vm *VirtualMachine) GetProvisioningState() string {
+	if vm.IsVirtualMachine() && vm.vm != nil &&
+		vm.vm.VirtualMachineProperties != nil &&
+		vm.vm.VirtualMachineProperties.ProvisioningState != nil {
+		return *vm.vm.VirtualMachineProperties.ProvisioningState
+	}
+	if vm.IsVirtualMachineScaleSetVM() &&
+		vm.vmssVM != nil &&
+		vm.vmssVM.VirtualMachineScaleSetVMProperties != nil &&
+		vm.vmssVM.VirtualMachineScaleSetVMProperties.ProvisioningState != nil {
+		return *vm.vmssVM.VirtualMachineScaleSetVMProperties.ProvisioningState
+	}
+	return consts.ProvisioningStateUnknown
+}
+
 // StringMap returns a map of strings built from the map of string pointers. The empty string is
 // used for nil pointers.
 func stringMap(msp map[string]*string) map[string]string {
diff --git a/pkg/util/string/string.go b/pkg/util/string/string.go
new file mode 100644
index 000000000..10510ecc7
--- /dev/null
+++ b/pkg/util/string/string.go
@@ -0,0 +1,24 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stringutils
+
+import "strings"
+
+// HasPrefixCaseInsensitive returns true if the string has the prefix, case-insensitive.
+func HasPrefixCaseInsensitive(s string, prefix string) bool {
+	return strings.HasPrefix(strings.ToLower(s), strings.ToLower(prefix))
+}
diff --git a/pkg/util/vm/vm.go b/pkg/util/vm/vm.go
new file mode 100644
index 000000000..3a13fb127
--- /dev/null
+++ b/pkg/util/vm/vm.go
@@ -0,0 +1,55 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vm
+
+import (
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute"
+
+	"k8s.io/klog/v2"
+	"k8s.io/utils/pointer"
+
+	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
+	stringutils "sigs.k8s.io/cloud-provider-azure/pkg/util/string"
+)
+
+// GetVMPowerState returns the power state of the VM
+func GetVMPowerState(vmName string, vmStatuses *[]compute.InstanceViewStatus) string {
+	logger := klog.Background().WithName("getVMSSVMPowerState").WithValues("vmName", vmName)
+	if vmStatuses != nil {
+		for _, status := range *vmStatuses {
+			state := pointer.StringDeref(status.Code, "")
+			if stringutils.HasPrefixCaseInsensitive(state, consts.VMPowerStatePrefix) {
+				return strings.TrimPrefix(state, consts.VMPowerStatePrefix)
+			}
+		}
+	}
+	logger.V(3).Info("vm status is nil in the instance view or there is no power state in the status")
+	return consts.VMPowerStateUnknown
+}
+
+// IsNotActiveVMState checks if the VM is in the active states
+func IsNotActiveVMState(provisioningState, powerState string) bool {
+	return strings.EqualFold(provisioningState, consts.ProvisioningStateDeleting) ||
+		strings.EqualFold(provisioningState, consts.ProvisioningStateUnknown) ||
+		strings.EqualFold(powerState, consts.VMPowerStateUnknown) ||
+		strings.EqualFold(powerState, consts.VMPowerStateStopped) ||
+		strings.EqualFold(powerState, consts.VMPowerStateStopping) ||
+		strings.EqualFold(powerState, consts.VMPowerStateDeallocated) ||
+		strings.EqualFold(powerState, consts.VMPowerStateDeallocating)
+}


Test patch:
diff --git a/pkg/provider/azure_backoff_test.go b/pkg/provider/azure_backoff_test.go
index f02b49b38..d2ea59829 100644
--- a/pkg/provider/azure_backoff_test.go
+++ b/pkg/provider/azure_backoff_test.go
@@ -576,7 +576,7 @@ func TestCreateOrUpdateVMSS(t *testing.T) {
 		{
 			vmss: compute.VirtualMachineScaleSet{
 				VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
-					ProvisioningState: pointer.String(consts.VirtualMachineScaleSetsDeallocating),
+					ProvisioningState: pointer.String(consts.ProvisioningStateDeleting),
 				},
 			},
 		},
diff --git a/pkg/provider/azure_standard_test.go b/pkg/provider/azure_standard_test.go
index 84872eaea..0e6b5e2f3 100644
--- a/pkg/provider/azure_standard_test.go
+++ b/pkg/provider/azure_standard_test.go
@@ -1305,7 +1305,7 @@ func TestGetStandardVMPowerStatusByNodeName(t *testing.T) {
 					ProvisioningState: pointer.String("Succeeded"),
 				},
 			},
-			expectedStatus: vmPowerStateUnknown,
+			expectedStatus: consts.VMPowerStateUnknown,
 		},
 		{
 			name:     "GetPowerStatusByNodeName should get vmPowerStateUnknown if vm.InstanceView.statuses is nil",
@@ -1317,7 +1317,7 @@ func TestGetStandardVMPowerStatusByNodeName(t *testing.T) {
 					InstanceView:      &compute.VirtualMachineInstanceView{},
 				},
 			},
-			expectedStatus: vmPowerStateUnknown,
+			expectedStatus: consts.VMPowerStateUnknown,
 		},
 	}
 	for _, test := range testcases {
diff --git a/pkg/provider/azure_vmss_test.go b/pkg/provider/azure_vmss_test.go
index 63687ee92..89ceff20f 100644
--- a/pkg/provider/azure_vmss_test.go
+++ b/pkg/provider/azure_vmss_test.go
@@ -790,7 +790,7 @@ func TestGetPowerStatusByNodeName(t *testing.T) {
 			description:        "GetPowerStatusByNodeName should return vmPowerStateUnknown when the vm.InstanceView.Statuses is nil",
 			vmList:             []string{"vmss-vm-000001"},
 			nilStatus:          true,
-			expectedPowerState: vmPowerStateUnknown,
+			expectedPowerState: consts.VMPowerStateUnknown,
 		},
 	}
 
@@ -2172,6 +2172,7 @@ func TestEnsureHostInPool(t *testing.T) {
 		isNilVMNetworkConfigs     bool
 		useMultipleSLBs           bool
 		isVMBeingDeleted          bool
+		isVMNotActive             bool
 		expectedNodeResourceGroup string
 		expectedVMSSName          string
 		expectedInstanceID        string
@@ -2261,6 +2262,11 @@ func TestEnsureHostInPool(t *testing.T) {
 				},
 			},
 		},
+		{
+			description:   "EnsureHostInPool should skip if the current node is not active",
+			nodeName:      "vmss-vm-000000",
+			isVMNotActive: true,
+		},
 	}
 
 	for _, test := range testCases {
@@ -2295,7 +2301,12 @@ func TestEnsureHostInPool(t *testing.T) {
 		if test.isNilVMNetworkConfigs {
 			expectedVMSSVMs[0].NetworkProfileConfiguration.NetworkInterfaceConfigurations = nil
 		}
-		mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
+		if test.isVMNotActive {
+			(*expectedVMSSVMs[0].InstanceView.Statuses)[0] = compute.InstanceViewStatus{
+				Code: pointer.String("PowerState/deallocated"),
+			}
+		}
+		mockVMSSVMClient := ss.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 		mockVMSSVMClient.EXPECT().List(
 			gomock.Any(),
 			ss.ResourceGroup,
@@ -2524,7 +2535,7 @@ func TestEnsureVMSSInPool(t *testing.T) {
 
 			expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, test.setIPv6Config)
 			if test.isVMSSDeallocating {
-				expectedVMSS.ProvisioningState = pointer.String(consts.VirtualMachineScaleSetsDeallocating)
+				expectedVMSS.ProvisioningState = pointer.String(consts.ProvisionStateDeleting)
 			}
 			if test.isVMSSNilNICConfig {
 				expectedVMSS.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = nil
@@ -2654,6 +2665,7 @@ func TestEnsureBackendPoolDeletedFromNodeCommon(t *testing.T) {
 		nodeName                  string
 		backendpoolIDs            []string
 		isNilVMNetworkConfigs     bool
+		isVMNotActive             bool
 		expectedNodeResourceGroup string
 		expectedVMSSName          string
 		expectedInstanceID        string
@@ -2716,6 +2728,11 @@ func TestEnsureBackendPoolDeletedFromNodeCommon(t *testing.T) {
 				},
 			},
 		},
+		{
+			description:   "ensureBackendPoolDeletedFromNode should skip if the node is not active",
+			nodeName:      "vmss-vm-000000",
+			isVMNotActive: true,
+		},
 	}
 
 	for _, test := range testCases {
@@ -2732,6 +2749,11 @@ func TestEnsureBackendPoolDeletedFromNodeCommon(t *testing.T) {
 			if test.isNilVMNetworkConfigs {
 				expectedVMSSVMs[0].NetworkProfileConfiguration.NetworkInterfaceConfigurations = nil
 			}
+			if test.isVMNotActive {
+				(*expectedVMSSVMs[0].InstanceView.Statuses)[0] = compute.InstanceViewStatus{
+					Code: pointer.String("PowerState/deallocated"),
+				}
+			}
 			mockVMSSVMClient := ss.cloud.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 			mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
 
@@ -2812,7 +2834,7 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) {
 
 		expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false)
 		if test.isVMSSDeallocating {
-			expectedVMSS.ProvisioningState = pointer.String(consts.VirtualMachineScaleSetsDeallocating)
+			expectedVMSS.ProvisioningState = pointer.String(consts.ProvisionStateDeleting)
 		}
 		if test.isVMSSNilNICConfig {
 			expectedVMSS.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = nil
diff --git a/pkg/provider/azure_vmssflex_test.go b/pkg/provider/azure_vmssflex_test.go
index a46f65b8b..c051559c6 100644
--- a/pkg/provider/azure_vmssflex_test.go
+++ b/pkg/provider/azure_vmssflex_test.go
@@ -1413,7 +1413,7 @@ func TestEnsureVMSSFlexInPool(t *testing.T) {
 		testVmssFlex := genreteTestVmssFlex("vmssflex1", testVmssFlex1ID)
 
 		if tc.isVMSSDeallocating {
-			testVmssFlex.ProvisioningState = pointer.String(consts.VirtualMachineScaleSetsDeallocating)
+			testVmssFlex.ProvisioningState = pointer.String(consts.ProvisionStateDeleting)
 		}
 		if !tc.hasDefaultVMProfile {
 			testVmssFlex.VirtualMachineProfile = nil
@@ -1671,7 +1671,7 @@ func TestEnsureBackendPoolDeletedFromVMSetsVmssFlex(t *testing.T) {
 			testVmssFlex := genreteTestVmssFlex("vmssflex1", testVmssFlex1ID)
 
 			if tc.isVMSSDeallocating {
-				testVmssFlex.ProvisioningState = pointer.String(consts.VirtualMachineScaleSetsDeallocating)
+				testVmssFlex.ProvisioningState = pointer.String(consts.ProvisionStateDeleting)
 			}
 			if !tc.hasDefaultVMProfile {
 				testVmssFlex.VirtualMachineProfile = nil
diff --git a/pkg/util/string/string_test.go b/pkg/util/string/string_test.go
new file mode 100644
index 000000000..8449cfc6f
--- /dev/null
+++ b/pkg/util/string/string_test.go
@@ -0,0 +1,42 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stringutils
+
+import (
+	"testing"
+)
+
+func TestHasPrefixCaseInsensitive(t *testing.T) {
+	cases := []struct {
+		s      string
+		prefix string
+		want   bool
+	}{
+		{"HelloWorld", "hello", true},
+		{"HelloWorld", "WORLD", false},
+		{"", "prefix", false},
+		{"CaseSensitive", "casesensitive", true},
+		{"CaseSensitive", "CASE", true},
+	}
+
+	for _, c := range cases {
+		got := HasPrefixCaseInsensitive(c.s, c.prefix)
+		if got != c.want {
+			t.Errorf("HasPrefixCaseInsensitive(%q, %q) == %v, want %v", c.s, c.prefix, got, c.want)
+		}
+	}
+}
diff --git a/pkg/util/vm/vm_test.go b/pkg/util/vm/vm_test.go
new file mode 100644
index 000000000..f61d869bd
--- /dev/null
+++ b/pkg/util/vm/vm_test.go
@@ -0,0 +1,134 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vm
+
+import (
+	"testing"
+
+	"k8s.io/utils/pointer"
+
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetVMPowerState(t *testing.T) {
+	type testCase struct {
+		name       string
+		vmStatuses *[]compute.InstanceViewStatus
+		expected   string
+	}
+
+	tests := []testCase{
+		{
+			name: "should return power state when there is power state status",
+			vmStatuses: &[]compute.InstanceViewStatus{
+				{Code: pointer.String("foo")},
+				{Code: pointer.String("PowerState/Running")},
+			},
+			expected: "Running",
+		},
+		{
+			name: "should return unknown when there is no power state status",
+			vmStatuses: &[]compute.InstanceViewStatus{
+				{Code: pointer.String("foo")},
+			},
+			expected: "unknown",
+		},
+		{
+			name:       "should return unknown when vmStatuses is nil",
+			vmStatuses: nil,
+			expected:   "unknown",
+		},
+		{
+			name:       "should return unknown when vmStatuses is empty",
+			vmStatuses: &[]compute.InstanceViewStatus{},
+			expected:   "unknown",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.expected, GetVMPowerState("vm", test.vmStatuses))
+		})
+	}
+}
+
+func TestIsNotActiveVMState(t *testing.T) {
+	type testCase struct {
+		name              string
+		provisioningState string
+		powerState        string
+		expected          bool
+	}
+
+	tests := []testCase{
+		{
+			name:              "should return true when provisioning state is deleting",
+			provisioningState: "deleting",
+			powerState:        "running",
+			expected:          true,
+		},
+		{
+			name:              "should return true when provisioning state is unknown",
+			provisioningState: "unknown",
+			powerState:        "running",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is unknown",
+			provisioningState: "running",
+			powerState:        "unknown",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is stopped",
+			provisioningState: "running",
+			powerState:        "stopped",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is stopping",
+			provisioningState: "running",
+			powerState:        "stopping",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is deallocated",
+			provisioningState: "running",
+			powerState:        "deallocated",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is deallocating",
+			provisioningState: "running",
+			powerState:        "deallocating",
+			expected:          true,
+		},
+		{
+			name:              "should return false when provisioning state is running and power state is running",
+			provisioningState: "running",
+			powerState:        "running",
+			expected:          false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.expected, IsNotActiveVMState(test.provisioningState, test.powerState))
+		})
+	}
+}