Repository: kubernetes-sigs/cloud-provider-azure
Pull request #5365 (2024-02-04 05:09:43)

[release-1.29] fix: Skip attaching/detaching vmss vm to lb backend pool if the vm is…
This is an automated cherry-pick of #5356

/assign lzhecheng

```release-note
fix: Skip attaching/detaching vmss vm to lb backend pool if the vm is not active.

We should not update the VM instance if its provisioning state or power state is not good. This will save a lot of api calls and reduce throttling issues.
```


Patch:
diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go
index 3b2448658..75bd1708b 100644
--- a/pkg/consts/consts.go
+++ b/pkg/consts/consts.go
@@ -113,8 +113,8 @@ const (
 	// StrRawVersion is the raw version string
 	StrRawVersion string = "raw"
 
-	// VirtualMachineScaleSetsDeallocating indicates VMSS instances are in Deallocating state.
-	VirtualMachineScaleSetsDeallocating = "Deallocating"
+	// ProvisionStateDeleting indicates VMSS instances are in Deleting state.
+	ProvisionStateDeleting = "Deleting"
 	// VmssMachineIDTemplate is the vmss manchine ID template
 	VmssMachineIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%s"
 	// VMSetCIDRIPV4TagKey specifies the node ipv4 CIDR mask of the instances on the VMSS or VMAS
@@ -132,6 +132,8 @@ const (
 	ProvisioningStateDeleting = "Deleting"
 	// ProvisioningStateSucceeded ...
 	ProvisioningStateSucceeded = "Succeeded"
+	// ProvisioningStateUnknown is the unknown provisioning state
+	ProvisioningStateUnknown = "Unknown"
 )
 
 // cache
@@ -572,3 +574,13 @@ const (
 	ClusterServiceLoadBalancerHealthProbeDefaultPath         = "/healthz"
 	SharedProbeName                                          = "cluster-service-shared-health-probe"
 )
+
+// VM power state
+const (
+	VMPowerStatePrefix       = "PowerState/"
+	VMPowerStateStopped      = "stopped"
+	VMPowerStateStopping     = "stopping"
+	VMPowerStateDeallocated  = "deallocated"
+	VMPowerStateDeallocating = "deallocating"
+	VMPowerStateUnknown      = "unknown"
+)
diff --git a/pkg/provider/azure_instances_v1.go b/pkg/provider/azure_instances_v1.go
index e8a4a76f4..2e5c9ebbf 100644
--- a/pkg/provider/azure_instances_v1.go
+++ b/pkg/provider/azure_instances_v1.go
@@ -35,12 +35,6 @@ import (
 var _ cloudprovider.Instances = (*Cloud)(nil)
 
 const (
-	vmPowerStatePrefix       = "PowerState/"
-	vmPowerStateStopped      = "stopped"
-	vmPowerStateDeallocated  = "deallocated"
-	vmPowerStateDeallocating = "deallocating"
-	vmPowerStateUnknown      = "unknown"
-
 	// nodeNameEnvironmentName is the environment variable name for getting node name.
 	// It is only used for out-of-tree cloud provider.
 	nodeNameEnvironmentName = "NODE_NAME"
@@ -266,7 +260,7 @@ func (az *Cloud) InstanceShutdownByProviderID(_ context.Context, providerID stri
 
 	status := strings.ToLower(powerStatus)
 	provisioningSucceeded := strings.EqualFold(strings.ToLower(provisioningState), strings.ToLower(string(consts.ProvisioningStateSucceeded)))
-	return provisioningSucceeded && (status == vmPowerStateStopped || status == vmPowerStateDeallocated || status == vmPowerStateDeallocating), nil
+	return provisioningSucceeded && (status == consts.VMPowerStateStopped || status == consts.VMPowerStateDeallocated || status == consts.VMPowerStateDeallocating), nil
 }
 
 func (az *Cloud) isCurrentInstance(name types.NodeName, metadataVMName string) (bool, error) {
diff --git a/pkg/provider/azure_standard.go b/pkg/provider/azure_standard.go
index 702435652..1d1863b2c 100644
--- a/pkg/provider/azure_standard.go
+++ b/pkg/provider/azure_standard.go
@@ -38,10 +38,12 @@ import (
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
+	vmutil "sigs.k8s.io/cloud-provider-azure/pkg/util/vm"
 )
 
 var (
@@ -501,19 +503,13 @@ func (as *availabilitySet) GetPowerStatusByNodeName(name string) (powerState str
 		return powerState, err
 	}
 
-	if vm.InstanceView != nil && vm.InstanceView.Statuses != nil {
-		statuses := *vm.InstanceView.Statuses
-		for _, status := range statuses {
-			state := pointer.StringDeref(status.Code, "")
-			if strings.HasPrefix(state, vmPowerStatePrefix) {
-				return strings.TrimPrefix(state, vmPowerStatePrefix), nil
-			}
-		}
+	if vm.InstanceView != nil {
+		return vmutil.GetVMPowerState(ptr.Deref(vm.Name, ""), vm.InstanceView.Statuses), nil
 	}
 
 	// vm.InstanceView or vm.InstanceView.Statuses are nil when the VM is under deleting.
 	klog.V(3).Infof("InstanceView for node %q is nil, assuming it's deleting", name)
-	return vmPowerStateUnknown, nil
+	return consts.VMPowerStateUnknown, nil
 }
 
 // GetProvisioningStateByNodeName returns the provisioningState for the specified node.
diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go
index 69389b263..3c84f767b 100644
--- a/pkg/provider/azure_vmss.go
+++ b/pkg/provider/azure_vmss.go
@@ -35,11 +35,13 @@ import (
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
 	"sigs.k8s.io/cloud-provider-azure/pkg/provider/virtualmachine"
+	vmutil "sigs.k8s.io/cloud-provider-azure/pkg/util/vm"
 )
 
 var (
@@ -282,20 +284,14 @@ func (ss *ScaleSet) GetPowerStatusByNodeName(name string) (powerState string, er
 
 	if vm.IsVirtualMachineScaleSetVM() {
 		v := vm.AsVirtualMachineScaleSetVM()
-		if v.InstanceView != nil && v.InstanceView.Statuses != nil {
-			statuses := *v.InstanceView.Statuses
-			for _, status := range statuses {
-				state := pointer.StringDeref(status.Code, "")
-				if strings.HasPrefix(state, vmPowerStatePrefix) {
-					return strings.TrimPrefix(state, vmPowerStatePrefix), nil
-				}
-			}
+		if v.InstanceView != nil {
+			return vmutil.GetVMPowerState(ptr.Deref(v.Name, ""), v.InstanceView.Statuses), nil
 		}
 	}
 
 	// vm.InstanceView or vm.InstanceView.Statuses are nil when the VM is under deleting.
 	klog.V(3).Infof("InstanceView for node %q is nil, assuming it's deleting", name)
-	return vmPowerStateUnknown, nil
+	return consts.VMPowerStateUnknown, nil
 }
 
 // GetProvisioningStateByNodeName returns the provisioningState for the specified node.
@@ -1029,6 +1025,8 @@ func getPrimaryIPConfigFromVMSSNetworkConfig(config *compute.VirtualMachineScale
 // EnsureHostInPool ensures the given VM's Primary NIC's Primary IP Configuration is
 // participating in the specified LoadBalancer Backend Pool, which returns (resourceGroup, vmasName, instanceID, vmssVM, error).
 func (ss *ScaleSet) EnsureHostInPool(_ *v1.Service, nodeName types.NodeName, backendPoolID string, vmSetNameOfLB string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
+	logger := klog.Background().WithName("EnsureHostInPool").
+		WithValues("nodeName", nodeName, "backendPoolID", backendPoolID, "vmSetNameOfLB", vmSetNameOfLB)
 	vmName := mapNodeNameToVMName(nodeName)
 	vm, err := ss.getVmssVM(vmName, azcache.CacheReadTypeDefault)
 	if err != nil {
@@ -1037,13 +1035,20 @@ func (ss *ScaleSet) EnsureHostInPool(_ *v1.Service, nodeName types.NodeName, bac
 			return "", "", "", nil, nil
 		}
 
-		klog.Errorf("EnsureHostInPool: failed to get VMSS VM %s: %v", vmName, err)
+		logger.Error(err, "failed to get vmss vm", "vmName", vmName)
 		if !errors.Is(err, ErrorNotVmssInstance) {
 			return "", "", "", nil, err
 		}
 	}
+	statuses := vm.GetInstanceViewStatus()
+	vmPowerState := vmutil.GetVMPowerState(vm.Name, statuses)
+	provisioningState := vm.GetProvisioningState()
+	if vmutil.IsNotActiveVMState(provisioningState, vmPowerState) {
+		logger.V(2).Info("skip updating the node because it is not in an active state", "vmName", vmName, "provisioningState", provisioningState, "vmPowerState", vmPowerState)
+		return "", "", "", nil, nil
+	}
 
-	klog.V(2).Infof("ensuring node %q of scaleset %q in LB backendpool %q", nodeName, vm.VMSSName, backendPoolID)
+	logger.V(2).Info("ensuring the vmss node in LB backendpool", "vmss name", vm.VMSSName)
 
 	// Check scale set name:
 	// - For basic SKU load balancer, return nil if the node's scale set is mismatched with vmSetNameOfLB.
@@ -1057,14 +1062,13 @@ func (ss *ScaleSet) EnsureHostInPool(_ *v1.Service, nodeName types.NodeName, bac
 	}
 
 	if vmSetNameOfLB != "" && needCheck && !strings.EqualFold(vmSetNameOfLB, vm.VMSSName) {
-		klog.V(3).Infof("EnsureHostInPool skips node %s because it is not in the ScaleSet %s", vmName, vmSetNameOfLB)
+		logger.V(3).Info("skips the node %s because it is not in the ScaleSet %s", vmName, vmSetNameOfLB)
 		return "", "", "", nil, nil
 	}
 
 	// Find primary network interface configuration.
 	if vm.VirtualMachineScaleSetVMProperties.NetworkProfileConfiguration.NetworkInterfaceConfigurations == nil {
-		klog.V(4).Infof("EnsureHostInPool: cannot obtain the primary network interface configuration, of vm %s, "+
-			"probably because the vm's being deleted", vmName)
+		logger.V(4).Info("cannot obtain the primary network interface configuration, of the vm, probably because the vm's being deleted", "vmName", vmName)
 		return "", "", "", nil, nil
 	}
 
@@ -1114,7 +1118,7 @@ func (ss *ScaleSet) EnsureHostInPool(_ *v1.Service, nodeName types.NodeName, bac
 			return "", "", "", nil, err
 		}
 		if !isSameLB {
-			klog.V(4).Infof("Node %q has already been added to LB %q, omit adding it to a new one", nodeName, oldLBName)
+			logger.V(4).Info("The node has already been added to an LB, omit adding it to a new one", "lbName", oldLBName)
 			return "", "", "", nil, nil
 		}
 	}
@@ -1221,7 +1225,7 @@ func (ss *ScaleSet) ensureVMSSInPool(_ *v1.Service, nodes []*v1.Node, backendPoo
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 			klog.V(3).Infof("ensureVMSSInPool: found vmss %s being deleted, skipping", vmssName)
 			continue
 		}
@@ -1493,6 +1497,7 @@ func (ss *ScaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node, bac
 // ensureBackendPoolDeletedFromNode ensures the loadBalancer backendAddressPools deleted
 // from the specified node, which returns (resourceGroup, vmasName, instanceID, vmssVM, error).
 func (ss *ScaleSet) ensureBackendPoolDeletedFromNode(nodeName string, backendPoolIDs []string) (string, string, string, *compute.VirtualMachineScaleSetVM, error) {
+	logger := klog.Background().WithName("ensureBackendPoolDeletedFromNode").WithValues("nodeName", nodeName, "backendPoolIDs", backendPoolIDs)
 	vm, err := ss.getVmssVM(nodeName, azcache.CacheReadTypeDefault)
 	if err != nil {
 		if errors.Is(err, cloudprovider.InstanceNotFound) {
@@ -1503,6 +1508,14 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromNode(nodeName string, backendPoo
 		return "", "", "", nil, err
 	}
 
+	statuses := vm.GetInstanceViewStatus()
+	vmPowerState := vmutil.GetVMPowerState(vm.Name, statuses)
+	provisioningState := vm.GetProvisioningState()
+	if vmutil.IsNotActiveVMState(provisioningState, vmPowerState) {
+		logger.V(2).Info("skip updating the node because it is not in an active state", "provisioningState", provisioningState, "vmPowerState", vmPowerState)
+		return "", "", "", nil, nil
+	}
+
 	// Find primary network interface configuration.
 	if vm.VirtualMachineScaleSetVMProperties.NetworkProfileConfiguration.NetworkInterfaceConfigurations == nil {
 		klog.V(4).Infof("ensureBackendPoolDeletedFromNode: cannot obtain the primary network interface configuration, of vm %s, "+
@@ -1679,7 +1692,7 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromVmssUniform(backendPoolIDs []str
 
 			// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 			// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-			if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+			if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 				klog.V(3).Infof("ensureBackendPoolDeletedFromVMSS: found vmss %s being deleted, skipping", pointer.StringDeref(vmss.Name, ""))
 				return true
 			}
@@ -2086,7 +2099,7 @@ func (ss *ScaleSet) EnsureBackendPoolDeletedFromVMSets(vmssNamesMap map[string]b
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 			klog.V(3).Infof("EnsureBackendPoolDeletedFromVMSets: found vmss %s being deleted, skipping", vmssName)
 			continue
 		}
diff --git a/pkg/provider/azure_vmss_repo.go b/pkg/provider/azure_vmss_repo.go
index 6721dceab..89b5a900f 100644
--- a/pkg/provider/azure_vmss_repo.go
+++ b/pkg/provider/azure_vmss_repo.go
@@ -39,7 +39,7 @@ func (az *Cloud) CreateOrUpdateVMSS(resourceGroupName string, VMScaleSetName str
 		klog.Errorf("CreateOrUpdateVMSS: error getting vmss(%s): %v", VMScaleSetName, rerr)
 		return rerr
 	}
-	if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+	if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 		klog.V(3).Infof("CreateOrUpdateVMSS: found vmss %s being deleted, skipping", VMScaleSetName)
 		return nil
 	}
diff --git a/pkg/provider/azure_vmssflex.go b/pkg/provider/azure_vmssflex.go
index ef98ae0eb..646785c75 100644
--- a/pkg/provider/azure_vmssflex.go
+++ b/pkg/provider/azure_vmssflex.go
@@ -27,16 +27,19 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-07-01/network"
+
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
+	vmutil "sigs.k8s.io/cloud-provider-azure/pkg/util/vm"
 )
 
 var (
@@ -267,19 +270,13 @@ func (fs *FlexScaleSet) GetPowerStatusByNodeName(name string) (powerState string
 		return powerState, err
 	}
 
-	if vm.InstanceView != nil && vm.InstanceView.Statuses != nil {
-		statuses := *vm.InstanceView.Statuses
-		for _, status := range statuses {
-			state := pointer.StringDeref(status.Code, "")
-			if strings.HasPrefix(state, vmPowerStatePrefix) {
-				return strings.TrimPrefix(state, vmPowerStatePrefix), nil
-			}
-		}
+	if vm.InstanceView != nil {
+		return vmutil.GetVMPowerState(ptr.Deref(vm.Name, ""), vm.InstanceView.Statuses), nil
 	}
 
 	// vm.InstanceView or vm.InstanceView.Statuses are nil when the VM is under deleting.
 	klog.V(3).Infof("InstanceView for node %q is nil, assuming it's deleting", name)
-	return vmPowerStateUnknown, nil
+	return consts.VMPowerStateUnknown, nil
 }
 
 // GetPrimaryInterface gets machine primary network interface by node name.
@@ -616,7 +613,7 @@ func (fs *FlexScaleSet) ensureVMSSFlexInPool(_ *v1.Service, nodes []*v1.Node, ba
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmssFlex.ProvisioningState != nil && strings.EqualFold(*vmssFlex.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmssFlex.ProvisioningState != nil && strings.EqualFold(*vmssFlex.ProvisioningState, consts.ProvisionStateDeleting) {
 			klog.V(3).Infof("ensureVMSSFlexInPool: found vmss %s being deleted, skipping", vmssFlexID)
 			continue
 		}
@@ -790,7 +787,7 @@ func (fs *FlexScaleSet) EnsureBackendPoolDeletedFromVMSets(vmssNamesMap map[stri
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.ProvisionStateDeleting) {
 			klog.V(3).Infof("fs.EnsureBackendPoolDeletedFromVMSets: found vmss %s being deleted, skipping", vmssName)
 			continue
 		}
diff --git a/pkg/provider/virtualmachine/virtualmachine.go b/pkg/provider/virtualmachine/virtualmachine.go
index bd6b77535..924f08c8e 100644
--- a/pkg/provider/virtualmachine/virtualmachine.go
+++ b/pkg/provider/virtualmachine/virtualmachine.go
@@ -18,7 +18,10 @@ package virtualmachine
 
 import (
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute"
+
 	"k8s.io/utils/pointer"
+
+	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 )
 
 type Variant string
@@ -145,6 +148,36 @@ func (vm *VirtualMachine) AsVirtualMachineScaleSetVM() *compute.VirtualMachineSc
 	return vm.vmssVM
 }
 
+func (vm *VirtualMachine) GetInstanceViewStatus() *[]compute.InstanceViewStatus {
+	if vm.IsVirtualMachine() && vm.vm != nil &&
+		vm.vm.VirtualMachineProperties != nil &&
+		vm.vm.VirtualMachineProperties.InstanceView != nil {
+		return vm.vm.VirtualMachineProperties.InstanceView.Statuses
+	}
+	if vm.IsVirtualMachineScaleSetVM() &&
+		vm.vmssVM != nil &&
+		vm.vmssVM.VirtualMachineScaleSetVMProperties != nil &&
+		vm.vmssVM.VirtualMachineScaleSetVMProperties.InstanceView != nil {
+		return vm.vmssVM.VirtualMachineScaleSetVMProperties.InstanceView.Statuses
+	}
+	return nil
+}
+
+func (vm *VirtualMachine) GetProvisioningState() string {
+	if vm.IsVirtualMachine() && vm.vm != nil &&
+		vm.vm.VirtualMachineProperties != nil &&
+		vm.vm.VirtualMachineProperties.ProvisioningState != nil {
+		return *vm.vm.VirtualMachineProperties.ProvisioningState
+	}
+	if vm.IsVirtualMachineScaleSetVM() &&
+		vm.vmssVM != nil &&
+		vm.vmssVM.VirtualMachineScaleSetVMProperties != nil &&
+		vm.vmssVM.VirtualMachineScaleSetVMProperties.ProvisioningState != nil {
+		return *vm.vmssVM.VirtualMachineScaleSetVMProperties.ProvisioningState
+	}
+	return consts.ProvisioningStateUnknown
+}
+
 // StringMap returns a map of strings built from the map of string pointers. The empty string is
 // used for nil pointers.
 func stringMap(msp map[string]*string) map[string]string {
diff --git a/pkg/util/string/string.go b/pkg/util/string/string.go
new file mode 100644
index 000000000..10510ecc7
--- /dev/null
+++ b/pkg/util/string/string.go
@@ -0,0 +1,24 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stringutils
+
+import "strings"
+
+// HasPrefixCaseInsensitive returns true if the string has the prefix, case-insensitive.
+func HasPrefixCaseInsensitive(s string, prefix string) bool {
+	return strings.HasPrefix(strings.ToLower(s), strings.ToLower(prefix))
+}
diff --git a/pkg/util/vm/vm.go b/pkg/util/vm/vm.go
new file mode 100644
index 000000000..a5084f30b
--- /dev/null
+++ b/pkg/util/vm/vm.go
@@ -0,0 +1,55 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vm
+
+import (
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute"
+
+	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
+
+	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
+	stringutils "sigs.k8s.io/cloud-provider-azure/pkg/util/string"
+)
+
+// GetVMPowerState returns the power state of the VM
+func GetVMPowerState(vmName string, vmStatuses *[]compute.InstanceViewStatus) string {
+	logger := klog.Background().WithName("getVMSSVMPowerState").WithValues("vmName", vmName)
+	if vmStatuses != nil {
+		for _, status := range *vmStatuses {
+			state := ptr.Deref(status.Code, "")
+			if stringutils.HasPrefixCaseInsensitive(state, consts.VMPowerStatePrefix) {
+				return strings.TrimPrefix(state, consts.VMPowerStatePrefix)
+			}
+		}
+	}
+	logger.V(3).Info("vm status is nil in the instance view or there is no power state in the status")
+	return consts.VMPowerStateUnknown
+}
+
+// IsNotActiveVMState checks if the VM is in the active states
+func IsNotActiveVMState(provisioningState, powerState string) bool {
+	return strings.EqualFold(provisioningState, consts.ProvisioningStateDeleting) ||
+		strings.EqualFold(provisioningState, consts.ProvisioningStateUnknown) ||
+		strings.EqualFold(powerState, consts.VMPowerStateUnknown) ||
+		strings.EqualFold(powerState, consts.VMPowerStateStopped) ||
+		strings.EqualFold(powerState, consts.VMPowerStateStopping) ||
+		strings.EqualFold(powerState, consts.VMPowerStateDeallocated) ||
+		strings.EqualFold(powerState, consts.VMPowerStateDeallocating)
+}


Test patch:
diff --git a/pkg/provider/azure_standard_test.go b/pkg/provider/azure_standard_test.go
index d63697928..9369480ad 100644
--- a/pkg/provider/azure_standard_test.go
+++ b/pkg/provider/azure_standard_test.go
@@ -1050,7 +1050,7 @@ func TestGetStandardVMPowerStatusByNodeName(t *testing.T) {
 					ProvisioningState: pointer.String("Succeeded"),
 				},
 			},
-			expectedStatus: vmPowerStateUnknown,
+			expectedStatus: consts.VMPowerStateUnknown,
 		},
 		{
 			name:     "GetPowerStatusByNodeName should get vmPowerStateUnknown if vm.InstanceView.statuses is nil",
@@ -1062,7 +1062,7 @@ func TestGetStandardVMPowerStatusByNodeName(t *testing.T) {
 					InstanceView:      &compute.VirtualMachineInstanceView{},
 				},
 			},
-			expectedStatus: vmPowerStateUnknown,
+			expectedStatus: consts.VMPowerStateUnknown,
 		},
 	}
 	for _, test := range testcases {
diff --git a/pkg/provider/azure_vmss_repo_test.go b/pkg/provider/azure_vmss_repo_test.go
index f7802d49b..a8f3fef7a 100644
--- a/pkg/provider/azure_vmss_repo_test.go
+++ b/pkg/provider/azure_vmss_repo_test.go
@@ -62,7 +62,7 @@ func TestCreateOrUpdateVMSS(t *testing.T) {
 		{
 			vmss: compute.VirtualMachineScaleSet{
 				VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
-					ProvisioningState: pointer.String(consts.VirtualMachineScaleSetsDeallocating),
+					ProvisioningState: pointer.String(consts.ProvisionStateDeleting),
 				},
 			},
 		},
diff --git a/pkg/provider/azure_vmss_test.go b/pkg/provider/azure_vmss_test.go
index 4617023e9..439308383 100644
--- a/pkg/provider/azure_vmss_test.go
+++ b/pkg/provider/azure_vmss_test.go
@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 
 	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/interfaceclient/mockinterfaceclient"
 	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/publicipclient/mockpublicipclient"
@@ -791,7 +792,7 @@ func TestGetPowerStatusByNodeName(t *testing.T) {
 			description:        "GetPowerStatusByNodeName should return vmPowerStateUnknown when the vm.InstanceView.Statuses is nil",
 			vmList:             []string{"vmss-vm-000001"},
 			nilStatus:          true,
-			expectedPowerState: vmPowerStateUnknown,
+			expectedPowerState: consts.VMPowerStateUnknown,
 		},
 	}
 
@@ -2096,6 +2097,7 @@ func TestEnsureHostInPool(t *testing.T) {
 		isBasicLB                 bool
 		isNilVMNetworkConfigs     bool
 		isVMBeingDeleted          bool
+		isVMNotActive             bool
 		expectedNodeResourceGroup string
 		expectedVMSSName          string
 		expectedInstanceID        string
@@ -2179,6 +2181,11 @@ func TestEnsureHostInPool(t *testing.T) {
 				},
 			},
 		},
+		{
+			description:   "EnsureHostInPool should skip if the current node is not active",
+			nodeName:      "vmss-vm-000000",
+			isVMNotActive: true,
+		},
 	}
 
 	for _, test := range testCases {
@@ -2209,6 +2216,11 @@ func TestEnsureHostInPool(t *testing.T) {
 		if test.isNilVMNetworkConfigs {
 			expectedVMSSVMs[0].NetworkProfileConfiguration.NetworkInterfaceConfigurations = nil
 		}
+		if test.isVMNotActive {
+			(*expectedVMSSVMs[0].InstanceView.Statuses)[0] = compute.InstanceViewStatus{
+				Code: ptr.To("PowerState/deallocated"),
+			}
+		}
 		mockVMSSVMClient := ss.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 		mockVMSSVMClient.EXPECT().List(
 			gomock.Any(),
@@ -2438,7 +2450,7 @@ func TestEnsureVMSSInPool(t *testing.T) {
 
 			expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, test.setIPv6Config)
 			if test.isVMSSDeallocating {
-				expectedVMSS.ProvisioningState = pointer.String(consts.VirtualMachineScaleSetsDeallocating)
+				expectedVMSS.ProvisioningState = pointer.String(consts.ProvisionStateDeleting)
 			}
 			if test.isVMSSNilNICConfig {
 				expectedVMSS.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = nil
@@ -2568,6 +2580,7 @@ func TestEnsureBackendPoolDeletedFromNodeCommon(t *testing.T) {
 		nodeName                  string
 		backendpoolIDs            []string
 		isNilVMNetworkConfigs     bool
+		isVMNotActive             bool
 		expectedNodeResourceGroup string
 		expectedVMSSName          string
 		expectedInstanceID        string
@@ -2630,6 +2643,11 @@ func TestEnsureBackendPoolDeletedFromNodeCommon(t *testing.T) {
 				},
 			},
 		},
+		{
+			description:   "ensureBackendPoolDeletedFromNode should skip if the node is not active",
+			nodeName:      "vmss-vm-000000",
+			isVMNotActive: true,
+		},
 	}
 
 	for _, test := range testCases {
@@ -2646,6 +2664,11 @@ func TestEnsureBackendPoolDeletedFromNodeCommon(t *testing.T) {
 			if test.isNilVMNetworkConfigs {
 				expectedVMSSVMs[0].NetworkProfileConfiguration.NetworkInterfaceConfigurations = nil
 			}
+			if test.isVMNotActive {
+				(*expectedVMSSVMs[0].InstanceView.Statuses)[0] = compute.InstanceViewStatus{
+					Code: ptr.To("PowerState/deallocated"),
+				}
+			}
 			mockVMSSVMClient := ss.VirtualMachineScaleSetVMsClient.(*mockvmssvmclient.MockInterface)
 			mockVMSSVMClient.EXPECT().List(gomock.Any(), ss.ResourceGroup, testVMSSName, gomock.Any()).Return(expectedVMSSVMs, nil).AnyTimes()
 
@@ -2726,7 +2749,7 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) {
 
 		expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false)
 		if test.isVMSSDeallocating {
-			expectedVMSS.ProvisioningState = pointer.String(consts.VirtualMachineScaleSetsDeallocating)
+			expectedVMSS.ProvisioningState = pointer.String(consts.ProvisionStateDeleting)
 		}
 		if test.isVMSSNilNICConfig {
 			expectedVMSS.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = nil
diff --git a/pkg/provider/azure_vmssflex_test.go b/pkg/provider/azure_vmssflex_test.go
index b508ba037..bb69070d0 100644
--- a/pkg/provider/azure_vmssflex_test.go
+++ b/pkg/provider/azure_vmssflex_test.go
@@ -1250,7 +1250,7 @@ func TestEnsureVMSSFlexInPool(t *testing.T) {
 		testVmssFlex := genreteTestVmssFlex("vmssflex1", testVmssFlex1ID)
 
 		if tc.isVMSSDeallocating {
-			testVmssFlex.ProvisioningState = pointer.String(consts.VirtualMachineScaleSetsDeallocating)
+			testVmssFlex.ProvisioningState = pointer.String(consts.ProvisionStateDeleting)
 		}
 		if !tc.hasDefaultVMProfile {
 			testVmssFlex.VirtualMachineProfile = nil
@@ -1500,7 +1500,7 @@ func TestEnsureBackendPoolDeletedFromVMSetsVmssFlex(t *testing.T) {
 			testVmssFlex := genreteTestVmssFlex("vmssflex1", testVmssFlex1ID)
 
 			if tc.isVMSSDeallocating {
-				testVmssFlex.ProvisioningState = pointer.String(consts.VirtualMachineScaleSetsDeallocating)
+				testVmssFlex.ProvisioningState = pointer.String(consts.ProvisionStateDeleting)
 			}
 			if !tc.hasDefaultVMProfile {
 				testVmssFlex.VirtualMachineProfile = nil
diff --git a/pkg/util/string/string_test.go b/pkg/util/string/string_test.go
new file mode 100644
index 000000000..8449cfc6f
--- /dev/null
+++ b/pkg/util/string/string_test.go
@@ -0,0 +1,42 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stringutils
+
+import (
+	"testing"
+)
+
+func TestHasPrefixCaseInsensitive(t *testing.T) {
+	cases := []struct {
+		s      string
+		prefix string
+		want   bool
+	}{
+		{"HelloWorld", "hello", true},
+		{"HelloWorld", "WORLD", false},
+		{"", "prefix", false},
+		{"CaseSensitive", "casesensitive", true},
+		{"CaseSensitive", "CASE", true},
+	}
+
+	for _, c := range cases {
+		got := HasPrefixCaseInsensitive(c.s, c.prefix)
+		if got != c.want {
+			t.Errorf("HasPrefixCaseInsensitive(%q, %q) == %v, want %v", c.s, c.prefix, got, c.want)
+		}
+	}
+}
diff --git a/pkg/util/vm/vm_test.go b/pkg/util/vm/vm_test.go
new file mode 100644
index 000000000..ec1cd9708
--- /dev/null
+++ b/pkg/util/vm/vm_test.go
@@ -0,0 +1,134 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vm
+
+import (
+	"testing"
+
+	"k8s.io/utils/ptr"
+
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetVMPowerState(t *testing.T) {
+	type testCase struct {
+		name       string
+		vmStatuses *[]compute.InstanceViewStatus
+		expected   string
+	}
+
+	tests := []testCase{
+		{
+			name: "should return power state when there is power state status",
+			vmStatuses: &[]compute.InstanceViewStatus{
+				{Code: ptr.To("foo")},
+				{Code: ptr.To("PowerState/Running")},
+			},
+			expected: "Running",
+		},
+		{
+			name: "should return unknown when there is no power state status",
+			vmStatuses: &[]compute.InstanceViewStatus{
+				{Code: ptr.To("foo")},
+			},
+			expected: "unknown",
+		},
+		{
+			name:       "should return unknown when vmStatuses is nil",
+			vmStatuses: nil,
+			expected:   "unknown",
+		},
+		{
+			name:       "should return unknown when vmStatuses is empty",
+			vmStatuses: &[]compute.InstanceViewStatus{},
+			expected:   "unknown",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.expected, GetVMPowerState("vm", test.vmStatuses))
+		})
+	}
+}
+
+func TestIsNotActiveVMState(t *testing.T) {
+	type testCase struct {
+		name              string
+		provisioningState string
+		powerState        string
+		expected          bool
+	}
+
+	tests := []testCase{
+		{
+			name:              "should return true when provisioning state is deleting",
+			provisioningState: "deleting",
+			powerState:        "running",
+			expected:          true,
+		},
+		{
+			name:              "should return true when provisioning state is unknown",
+			provisioningState: "unknown",
+			powerState:        "running",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is unknown",
+			provisioningState: "running",
+			powerState:        "unknown",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is stopped",
+			provisioningState: "running",
+			powerState:        "stopped",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is stopping",
+			provisioningState: "running",
+			powerState:        "stopping",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is deallocated",
+			provisioningState: "running",
+			powerState:        "deallocated",
+			expected:          true,
+		},
+		{
+			name:              "should return true when power state is deallocating",
+			provisioningState: "running",
+			powerState:        "deallocating",
+			expected:          true,
+		},
+		{
+			name:              "should return false when provisioning state is running and power state is running",
+			provisioningState: "running",
+			powerState:        "running",
+			expected:          false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.expected, IsNotActiveVMState(test.provisioningState, test.powerState))
+		})
+	}
+}