Repository: kubernetes-sigs/cloud-provider-azure
Pull request #2245 (2022-08-31 21:17:11)

VMSS Flex Support: Part 7 of VMSet implementation: removing LB backend pool from VMSS Flex VM related functions

<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?
/kind feature
<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

#### What this PR does / why we need it:
This PR is the Part 7 of VMSet implementation, mainly focus on removing LB backend pool from VMSS Flex VM related functions
#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Refer to #639 

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
NONE
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```



Patch:
diff --git a/pkg/provider/azure_vmssflex.go b/pkg/provider/azure_vmssflex.go
index 3c2dbcbb0..74c8894a0 100644
--- a/pkg/provider/azure_vmssflex.go
+++ b/pkg/provider/azure_vmssflex.go
@@ -776,11 +776,245 @@ func (fs *FlexScaleSet) EnsureHostsInPool(service *v1.Service, nodes []*v1.Node,
 
 // EnsureBackendPoolDeletedFromVMSets ensures the loadBalancer backendAddressPools deleted from the specified VMSS Flex
 func (fs *FlexScaleSet) EnsureBackendPoolDeletedFromVMSets(vmssNamesMap map[string]bool, backendPoolID string) error {
+	vmssUpdaters := make([]func() error, 0, len(vmssNamesMap))
+	errors := make([]error, 0, len(vmssNamesMap))
+	for vmssName := range vmssNamesMap {
+		vmssName := vmssName
+		vmss, err := fs.getVmssFlexByName(vmssName)
+		if err != nil {
+			klog.Errorf("EnsureBackendPoolDeletedFromVMSets: failed to get VMSS %s: %v", vmssName, err)
+			errors = append(errors, err)
+			continue
+		}
+
+		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
+		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
+		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+			klog.V(3).Infof("EnsureBackendPoolDeletedFromVMSets: found vmss %s being deleted, skipping", vmssName)
+			continue
+		}
+		if vmss.VirtualMachineProfile == nil || vmss.VirtualMachineProfile.NetworkProfile == nil || vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations == nil {
+			klog.V(4).Infof("EnsureBackendPoolDeletedFromVMSets: cannot obtain the primary network interface configurations, of vmss %s", vmssName)
+			continue
+		}
+		vmssNIC := *vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations
+		primaryNIC, err := getPrimaryNetworkInterfaceConfigurationForScaleSet(vmssNIC, vmssName)
+		if err != nil {
+			klog.Errorf("EnsureBackendPoolDeletedFromVMSets: failed to get the primary network interface config of the VMSS %s: %v", vmssName, err)
+			errors = append(errors, err)
+			continue
+		}
+		primaryIPConfig, err := getPrimaryIPConfigFromVMSSNetworkConfig(primaryNIC)
+		if err != nil {
+			klog.Errorf("EnsureBackendPoolDeletedFromVMSets: failed to the primary IP config from the VMSS %s's network config : %v", vmssName, err)
+			errors = append(errors, err)
+			continue
+		}
+		loadBalancerBackendAddressPools := []compute.SubResource{}
+		if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
+			loadBalancerBackendAddressPools = *primaryIPConfig.LoadBalancerBackendAddressPools
+		}
+
+		var found bool
+		var newBackendPools []compute.SubResource
+		for i := len(loadBalancerBackendAddressPools) - 1; i >= 0; i-- {
+			curPool := loadBalancerBackendAddressPools[i]
+			if strings.EqualFold(backendPoolID, *curPool.ID) {
+				klog.V(10).Infof("EnsureBackendPoolDeletedFromVMSets gets unwanted backend pool %q for VMSS %s", backendPoolID, vmssName)
+				found = true
+				newBackendPools = append(loadBalancerBackendAddressPools[:i], loadBalancerBackendAddressPools[i+1:]...)
+			}
+		}
+		if !found {
+			continue
+		}
+
+		vmssUpdaters = append(vmssUpdaters, func() error {
+			// Compose a new vmss with added backendPoolID.
+			primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools
+			newVMSS := compute.VirtualMachineScaleSet{
+				Location: vmss.Location,
+				VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
+					VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{
+						NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{
+							NetworkInterfaceConfigurations: &vmssNIC,
+						},
+					},
+				},
+			}
+
+			klog.V(2).Infof("ensureBackendPoolDeletedFromVMSS begins to update vmss(%s) with backendPoolID %s", vmssName, backendPoolID)
+			rerr := fs.CreateOrUpdateVMSS(fs.ResourceGroup, vmssName, newVMSS)
+			if rerr != nil {
+				klog.Errorf("ensureBackendPoolDeletedFromVMSS CreateOrUpdateVMSS(%s) with new backendPoolID %s, err: %v", vmssName, backendPoolID, rerr)
+				return rerr.Error()
+			}
+
+			return nil
+		})
+	}
+
+	errs := utilerrors.AggregateGoroutines(vmssUpdaters...)
+	if errs != nil {
+		return utilerrors.Flatten(errs)
+	}
+	// Fail if there are other errors.
+	if len(errors) > 0 {
+		return utilerrors.Flatten(utilerrors.NewAggregate(errors))
+	}
+
 	return nil
 }
 
 // EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified nodes.
 func (fs *FlexScaleSet) EnsureBackendPoolDeleted(service *v1.Service, backendPoolID, vmSetName string, backendAddressPools *[]network.BackendAddressPool, deleteFromVMSet bool) error {
+	// Returns nil if backend address pools already deleted.
+	if backendAddressPools == nil {
+		return nil
+	}
+
+	mc := metrics.NewMetricContext("services", "vmssflex_ensure_backend_pool_deleted", fs.ResourceGroup, fs.SubscriptionID, getServiceName(service))
+	isOperationSucceeded := false
+	defer func() {
+		mc.ObserveOperationWithResult(isOperationSucceeded)
+	}()
+
+	ipConfigurationIDs := []string{}
+	for _, backendPool := range *backendAddressPools {
+		if strings.EqualFold(to.String(backendPool.ID), backendPoolID) && backendPool.BackendAddressPoolPropertiesFormat != nil && backendPool.BackendIPConfigurations != nil {
+			for _, ipConf := range *backendPool.BackendIPConfigurations {
+				if ipConf.ID == nil {
+					continue
+				}
+
+				ipConfigurationIDs = append(ipConfigurationIDs, *ipConf.ID)
+			}
+		}
+	}
+
+	vmssFlexNamesMap := make(map[string]bool)
+	vmssFlexVMNameMap := make(map[string]string)
+	allErrs := make([]error, 0)
+	for i := range ipConfigurationIDs {
+		ipConfigurationID := ipConfigurationIDs[i]
+		nodeName, vmssFlexName, nicName, err := fs.getNodeInformationByIPConfigurationID(ipConfigurationID)
+		if err != nil {
+			continue
+		}
+		if nodeName == "" {
+			continue
+		}
+		resourceGroupName, err := fs.GetNodeResourceGroup(nodeName)
+		if err != nil {
+			continue
+		}
+		// only vmsses in the resource group same as it's in azure config are included
+		if strings.EqualFold(resourceGroupName, fs.ResourceGroup) {
+			if fs.useStandardLoadBalancer() && !fs.EnableMultipleStandardLoadBalancers {
+				vmssFlexNamesMap[vmssFlexName] = true
+				vmssFlexVMNameMap[nodeName] = nicName
+			} else {
+				if strings.EqualFold(vmssFlexName, vmSetName) {
+					vmssFlexNamesMap[vmSetName] = true
+					vmssFlexVMNameMap[nodeName] = nicName
+				} else {
+					// Only remove nodes belonging to specified vmSet.
+					continue
+				}
+			}
+
+		}
+	}
+
+	// 1. Ensure the backendPoolID is deleted from the VMSS.
+	klog.V(2).Infof("1. Ensure the backendPoolID is deleted from the VMSS.")
+	if deleteFromVMSet {
+		err := fs.EnsureBackendPoolDeletedFromVMSets(vmssFlexNamesMap, backendPoolID)
+		if err != nil {
+			allErrs = append(allErrs, err)
+		}
+	}
+
+	klog.V(2).Infof("2. Ensure the backendPoolID is deleted from the VMSS VMs.")
+	// 2. Ensure the backendPoolID is deleted from the VMSS VMs.
+	klog.V(2).Infof("go into fs.ensureBackendPoolDeletedFromNode, vmssFlexVMNameMap: %s, size: %s", vmssFlexVMNameMap, len(vmssFlexVMNameMap))
+	err := fs.ensureBackendPoolDeletedFromNode(vmssFlexVMNameMap, backendPoolID)
+	klog.V(2).Infof("exit from fs.ensureBackendPoolDeletedFromNode")
+	if err != nil {
+		allErrs = append(allErrs, err)
+	}
+
+	if len(allErrs) > 0 {
+		return utilerrors.Flatten(utilerrors.NewAggregate(allErrs))
+	}
+
+	isOperationSucceeded = true
 	return nil
 
 }
+
+func (fs *FlexScaleSet) ensureBackendPoolDeletedFromNode(vmssFlexVMNameMap map[string]string, backendPoolID string) error {
+	nicUpdaters := make([]func() error, 0)
+	allErrs := make([]error, 0)
+	i := 0
+	for nodeName, nicName := range vmssFlexVMNameMap {
+		i++
+		klog.V(2).Infof("i = %s", i)
+
+		ctx, cancel := getContextWithCancel()
+		defer cancel()
+		nic, rerr := fs.InterfacesClient.Get(ctx, fs.ResourceGroup, nicName, "")
+		if rerr != nil {
+			return fmt.Errorf("ensureBackendPoolDeletedFromNode: failed to get interface of name %s: %w", nicName, rerr.Error())
+		}
+
+		if nic.ProvisioningState == consts.NicFailedState {
+			klog.Warningf("EnsureBackendPoolDeleted skips node %s because its primary nic %s is in Failed state", nodeName, *nic.Name)
+			continue
+		}
+
+		if nic.InterfacePropertiesFormat != nil && nic.InterfacePropertiesFormat.IPConfigurations != nil {
+			newIPConfigs := *nic.IPConfigurations
+			for j, ipConf := range newIPConfigs {
+				if !to.Bool(ipConf.Primary) {
+					continue
+				}
+				// found primary ip configuration
+				if ipConf.LoadBalancerBackendAddressPools != nil {
+					newLBAddressPools := *ipConf.LoadBalancerBackendAddressPools
+					for k := len(newLBAddressPools) - 1; k >= 0; k-- {
+						pool := newLBAddressPools[k]
+						if strings.EqualFold(to.String(pool.ID), backendPoolID) {
+							newLBAddressPools = append(newLBAddressPools[:k], newLBAddressPools[k+1:]...)
+							break
+						}
+					}
+					newIPConfigs[j].LoadBalancerBackendAddressPools = &newLBAddressPools
+				}
+			}
+			nic.IPConfigurations = &newIPConfigs
+
+			nicUpdaters = append(nicUpdaters, func() error {
+				ctx, cancel := getContextWithCancel()
+				defer cancel()
+				klog.V(2).Infof("EnsureBackendPoolDeleted begins to CreateOrUpdate for NIC(%s, %s) with backendPoolID %s", fs.resourceGroup, to.String(nic.Name), backendPoolID)
+				rerr := fs.InterfacesClient.CreateOrUpdate(ctx, fs.ResourceGroup, to.String(nic.Name), nic)
+				if rerr != nil {
+					klog.Errorf("EnsureBackendPoolDeleted CreateOrUpdate for NIC(%s, %s) failed with error %v", fs.resourceGroup, to.String(nic.Name), rerr.Error())
+					return rerr.Error()
+				}
+				klog.V(2).Infof("EnsureBackendPoolDeleted done")
+				return nil
+			})
+		}
+	}
+	klog.V(2).Infof("nicUpdaters size: %s", len(nicUpdaters))
+	errs := utilerrors.AggregateGoroutines(nicUpdaters...)
+	if errs != nil {
+		allErrs = append(allErrs, utilerrors.Flatten(errs))
+	}
+	if len(allErrs) > 0 {
+		return utilerrors.Flatten(utilerrors.NewAggregate(allErrs))
+	}
+	return nil
+}


Test patch:
diff --git a/pkg/provider/azure_vmssflex_test.go b/pkg/provider/azure_vmssflex_test.go
index adc816b26..cdfc60bc8 100644
--- a/pkg/provider/azure_vmssflex_test.go
+++ b/pkg/provider/azure_vmssflex_test.go
@@ -59,6 +59,19 @@ var (
 	testIPConfigurationID = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/testvm1-nic/ipConfigurations/pipConfig"
 	testBackendPoolID0    = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0"
 
+	testBackendPools = &[]network.BackendAddressPool{
+		{
+			ID: to.StringPtr(testBackendPoolID0),
+			BackendAddressPoolPropertiesFormat: &network.BackendAddressPoolPropertiesFormat{
+				BackendIPConfigurations: &[]network.InterfaceIPConfiguration{
+					{
+						ID: to.StringPtr(testIPConfigurationID),
+					},
+				},
+			},
+		},
+	}
+
 	testNic1 = generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded)
 
 	testNic2 = generateTestNic("testvm2-nic", true, network.ProvisioningStateSucceeded)
@@ -72,6 +85,7 @@ func generateTestNic(nicName string, isIPConfigurationsNil bool, provisioningSta
 			IPConfigurations: &[]network.InterfaceIPConfiguration{
 				{
 					InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
+						Primary:          to.BoolPtr(true),
 						PrivateIPAddress: to.StringPtr(nicName + "testPrivateIP"),
 						LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
 							{
@@ -1534,9 +1548,373 @@ func TestEnsureHostsInPoolVmssFlex(t *testing.T) {
 }
 
 func TestEnsureBackendPoolDeletedFromVMSetsVmssFlex(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	testCases := []struct {
+		description         string
+		vmssNamesMap        map[string]bool
+		backendPoolID       string
+		isVMSSDeallocating  bool
+		hasDefaultVMProfile bool
+		isNicConfigEmpty    bool
+		isIPConfigEmpty     bool
+		vmssPutErr          *retry.Error
+		expectedErr         error
+	}{
+		{
+			description: "EnsureBackendPoolDeletedFromVMSets should remove a backend pool from the vmss",
+			vmssNamesMap: map[string]bool{
+				"vmssflex1": true,
+			},
+			backendPoolID:       "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			hasDefaultVMProfile: true,
+			expectedErr:         nil,
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromVMSets should return error if the vmss does not exist",
+			vmssNamesMap: map[string]bool{
+				"NonExistingVmssflex": true,
+			},
+			backendPoolID:       "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			hasDefaultVMProfile: true,
+			expectedErr:         cloudprovider.InstanceNotFound,
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromVMSets should skip the vmss if it is deallocating",
+			vmssNamesMap: map[string]bool{
+				"vmssflex1": true,
+			},
+			backendPoolID:       "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			isVMSSDeallocating:  true,
+			hasDefaultVMProfile: true,
+			expectedErr:         nil,
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromVMSets should skip the vmss does not have default VM profile",
+			vmssNamesMap: map[string]bool{
+				"vmssflex1": true,
+			},
+			backendPoolID:       "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			isVMSSDeallocating:  false,
+			hasDefaultVMProfile: false,
+			expectedErr:         nil,
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromVMSets should skip the vmss does not have default VM profile",
+			vmssNamesMap: map[string]bool{
+				"vmssflex1": true,
+			},
+			backendPoolID:       "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			isVMSSDeallocating:  false,
+			hasDefaultVMProfile: true,
+			isNicConfigEmpty:    true,
+			expectedErr:         fmt.Errorf("failed to find a primary network configuration for the scale set \"vmssflex1\""),
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromVMSets should skip the vmss does not have default VM profile",
+			vmssNamesMap: map[string]bool{
+				"vmssflex1": true,
+			},
+			backendPoolID:       "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			isVMSSDeallocating:  false,
+			hasDefaultVMProfile: true,
+			isNicConfigEmpty:    false,
+			isIPConfigEmpty:     true,
+			expectedErr:         fmt.Errorf("failed to find a primary IP configuration"),
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromVMSets should skip the vmss if the backend pool is not in the vmss's backend pool list",
+			vmssNamesMap: map[string]bool{
+				"vmssflex1": true,
+			},
+			backendPoolID:       "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-1",
+			hasDefaultVMProfile: true,
+			expectedErr:         nil,
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromVMSets should skip the vmss if the backend pool is not in the vmss's backend pool list",
+			vmssNamesMap: map[string]bool{
+				"vmssflex1": true,
+			},
+			backendPoolID:       "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			hasDefaultVMProfile: true,
+			vmssPutErr:          &retry.Error{RawError: fmt.Errorf("failed to update nic")},
+			expectedErr:         fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: failed to update nic"),
+		},
+	}
+
+	for _, tc := range testCases {
+		fs, err := NewTestFlexScaleSet(ctrl)
+		assert.NoError(t, err, "unexpected error when creating test FlexScaleSet")
+
+		testVmssFlex := genreteTestVmssFlex()
+
+		if tc.isVMSSDeallocating {
+			testVmssFlex.ProvisioningState = to.StringPtr(consts.VirtualMachineScaleSetsDeallocating)
+		}
+		if !tc.hasDefaultVMProfile {
+			testVmssFlex.VirtualMachineProfile = nil
+		}
+		if tc.isNicConfigEmpty {
+			testVmssFlex.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = &[]compute.VirtualMachineScaleSetNetworkConfiguration{}
+		}
+		if tc.isIPConfigEmpty {
+			(*testVmssFlex.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations)[0].IPConfigurations = &[]compute.VirtualMachineScaleSetIPConfiguration{}
+		}
+
+		vmssFlexList := []compute.VirtualMachineScaleSet{testVmssFlex}
+
+		mockVMSSClient := fs.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
+		mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(vmssFlexList, nil).AnyTimes()
+		mockVMSSClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(testVmssFlex1, nil).AnyTimes()
+		mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.vmssPutErr).AnyTimes()
+
+		err = fs.EnsureBackendPoolDeletedFromVMSets(tc.vmssNamesMap, tc.backendPoolID)
+
+		if tc.expectedErr != nil {
+			assert.EqualError(t, err, tc.expectedErr.Error(), tc.description)
+		} else {
+			assert.NoError(t, err, tc.description)
+		}
+	}
+
+}
+
+func TestEnsureBackendPoolDeletedFromNodeVmssFlex(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	testCases := []struct {
+		description       string
+		vmssFlexVMNameMap map[string]string
+		backendPoolID     string
+		nic               network.Interface
+		nicGetErr         *retry.Error
+		nicPutErr         *retry.Error
+		expectedErr       error
+	}{
+		{
+			description: "EnsureBackendPoolDeletedFromNode should remove a backend pool from the vmss flex vm",
+			vmssFlexVMNameMap: map[string]string{
+				"vmssflex1000001": "testvm1-nic",
+			},
+			backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			nic:           generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded),
+			nicGetErr:     nil,
+			expectedErr:   nil,
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromNode should remove a backend pool from the vmss flex vm",
+			vmssFlexVMNameMap: map[string]string{
+				"vmssflex1000001": "testvm1-nic",
+			},
+			backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			nic:           generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded),
+			nicGetErr:     &retry.Error{RawError: fmt.Errorf("failed to get nic")},
+			expectedErr:   fmt.Errorf("ensureBackendPoolDeletedFromNode: failed to get interface of name testvm1-nic: Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: failed to get nic"),
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromNode should skip the node if the NIC is in failed state",
+			vmssFlexVMNameMap: map[string]string{
+				"vmssflex1000001": "testvm1-nic",
+			},
+			backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			nic:           generateTestNic("testvm1-nic", false, network.ProvisioningStateFailed),
+			nicGetErr:     nil,
+			expectedErr:   nil,
+		},
+		{
+			description: "EnsureBackendPoolDeletedFromNode should return error if NIC update fails",
+			vmssFlexVMNameMap: map[string]string{
+				"vmssflex1000001": "testvm1-nic",
+			},
+			backendPoolID: "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/lb/backendAddressPools/backendpool-0",
+			nic:           generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded),
+			nicGetErr:     nil,
+			nicPutErr:     &retry.Error{RawError: fmt.Errorf("failed to update nic")},
+			expectedErr:   fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: failed to update nic"),
+		},
+	}
+
+	for _, tc := range testCases {
+		fs, err := NewTestFlexScaleSet(ctrl)
+		assert.NoError(t, err, "unexpected error when creating test FlexScaleSet")
+
+		mockInterfacesClient := fs.InterfacesClient.(*mockinterfaceclient.MockInterface)
+		mockInterfacesClient.EXPECT().Get(gomock.Any(), gomock.Any(), "testvm1-nic", gomock.Any()).Return(tc.nic, tc.nicGetErr).AnyTimes()
+		mockInterfacesClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.nicPutErr).AnyTimes()
+
+		err = fs.ensureBackendPoolDeletedFromNode(tc.vmssFlexVMNameMap, tc.backendPoolID)
+
+		if tc.expectedErr != nil {
+			assert.EqualError(t, err, tc.expectedErr.Error(), tc.description)
+		} else {
+			assert.NoError(t, err, tc.description)
+		}
+	}
 
 }
 
 func TestEnsureBackendPoolDeletedVmssFlex(t *testing.T) {
 
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	testCases := []struct {
+		description         string
+		service             *v1.Service
+		vmSetName           string
+		backendPoolID       string
+		backendAddressPools *[]network.BackendAddressPool
+		deleteFromVMSet     bool
+		isStandardLB        bool
+		useMultipleSLBs     bool
+
+		testVMListWithoutInstanceView  []compute.VirtualMachine
+		testVMListWithOnlyInstanceView []compute.VirtualMachine
+		vmListErr                      error
+		nic                            network.Interface
+		nicGetErr                      *retry.Error
+		nicPutErr                      *retry.Error
+		vmssPutErr                     *retry.Error
+
+		expectedErr error
+	}{
+		{
+			description:                    "EnsureBackendPoolDeleted should delete a backend pool from the vm and vmss",
+			service:                        &v1.Service{},
+			vmSetName:                      "vmssflex1",
+			backendPoolID:                  testBackendPoolID0,
+			backendAddressPools:            testBackendPools,
+			deleteFromVMSet:                true,
+			isStandardLB:                   true,
+			useMultipleSLBs:                false,
+			testVMListWithoutInstanceView:  testVMListWithoutInstanceView,
+			testVMListWithOnlyInstanceView: testVMListWithOnlyInstanceView,
+			vmListErr:                      nil,
+			nic:                            generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded),
+			nicGetErr:                      nil,
+			expectedErr:                    nil,
+		},
+		{
+			description:                    "EnsureBackendPoolDeleted should do nothing if the backendPools is nil",
+			service:                        &v1.Service{},
+			vmSetName:                      "vmssflex1",
+			backendPoolID:                  testBackendPoolID0,
+			backendAddressPools:            nil,
+			deleteFromVMSet:                true,
+			isStandardLB:                   true,
+			useMultipleSLBs:                false,
+			testVMListWithoutInstanceView:  testVMListWithoutInstanceView,
+			testVMListWithOnlyInstanceView: testVMListWithOnlyInstanceView,
+			vmListErr:                      nil,
+			nic:                            generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded),
+			nicGetErr:                      nil,
+			expectedErr:                    nil,
+		},
+		{
+			description:                    "EnsureBackendPoolDeleted should delete a backend pool if the VMSetName matches nic's vmss",
+			service:                        &v1.Service{},
+			vmSetName:                      "vmssflex1",
+			backendPoolID:                  testBackendPoolID0,
+			backendAddressPools:            testBackendPools,
+			deleteFromVMSet:                true,
+			isStandardLB:                   true,
+			useMultipleSLBs:                true,
+			testVMListWithoutInstanceView:  testVMListWithoutInstanceView,
+			testVMListWithOnlyInstanceView: testVMListWithOnlyInstanceView,
+			vmListErr:                      nil,
+			nic:                            generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded),
+			nicGetErr:                      nil,
+			expectedErr:                    nil,
+		},
+		{
+			description:                    "EnsureBackendPoolDeleted should do nothing if the VMSetName does not match nic's vmss",
+			service:                        &v1.Service{},
+			vmSetName:                      "vmss",
+			backendPoolID:                  testBackendPoolID0,
+			backendAddressPools:            testBackendPools,
+			deleteFromVMSet:                true,
+			isStandardLB:                   true,
+			useMultipleSLBs:                true,
+			testVMListWithoutInstanceView:  testVMListWithoutInstanceView,
+			testVMListWithOnlyInstanceView: testVMListWithOnlyInstanceView,
+			vmListErr:                      nil,
+			nic:                            generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded),
+			nicGetErr:                      nil,
+			expectedErr:                    nil,
+		},
+		{
+			description:                    "EnsureBackendPoolDeleted should return error if vmss update fails",
+			service:                        &v1.Service{},
+			vmSetName:                      "vmssflex1",
+			backendPoolID:                  testBackendPoolID0,
+			backendAddressPools:            testBackendPools,
+			deleteFromVMSet:                true,
+			isStandardLB:                   true,
+			useMultipleSLBs:                false,
+			testVMListWithoutInstanceView:  testVMListWithoutInstanceView,
+			testVMListWithOnlyInstanceView: testVMListWithOnlyInstanceView,
+			vmListErr:                      nil,
+			nic:                            generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded),
+			nicGetErr:                      nil,
+			vmssPutErr:                     &retry.Error{RawError: fmt.Errorf("failed to update nic")},
+			expectedErr:                    fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: failed to update nic"),
+		},
+		{
+			description:                    "EnsureBackendPoolDeleted should return error if vmss update fails",
+			service:                        &v1.Service{},
+			vmSetName:                      "vmssflex1",
+			backendPoolID:                  testBackendPoolID0,
+			backendAddressPools:            testBackendPools,
+			deleteFromVMSet:                true,
+			isStandardLB:                   true,
+			useMultipleSLBs:                false,
+			testVMListWithoutInstanceView:  testVMListWithoutInstanceView,
+			testVMListWithOnlyInstanceView: testVMListWithOnlyInstanceView,
+			vmListErr:                      nil,
+			nic:                            generateTestNic("testvm1-nic", false, network.ProvisioningStateSucceeded),
+			nicGetErr:                      nil,
+			nicPutErr:                      &retry.Error{RawError: fmt.Errorf("failed to update nic")},
+			expectedErr:                    fmt.Errorf("Retriable: false, RetryAfter: 0s, HTTPStatusCode: 0, RawError: failed to update nic"),
+		},
+	}
+
+	for _, tc := range testCases {
+		fs, err := NewTestFlexScaleSet(ctrl)
+		assert.NoError(t, err, "unexpected error when creating test FlexScaleSet")
+		if tc.isStandardLB {
+			fs.Config.LoadBalancerSku = consts.LoadBalancerSkuStandard
+		}
+
+		if tc.useMultipleSLBs {
+			fs.EnableMultipleStandardLoadBalancers = true
+		}
+
+		testVmssFlex := genreteTestVmssFlex()
+		vmssFlexList := []compute.VirtualMachineScaleSet{testVmssFlex}
+
+		mockVMSSClient := fs.cloud.VirtualMachineScaleSetsClient.(*mockvmssclient.MockInterface)
+		mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return(vmssFlexList, nil).AnyTimes()
+		mockVMSSClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(testVmssFlex1, nil).AnyTimes()
+		mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.vmssPutErr).AnyTimes()
+
+		mockVMClient := fs.VirtualMachinesClient.(*mockvmclient.MockInterface)
+		mockVMClient.EXPECT().ListVmssFlexVMsWithoutInstanceView(gomock.Any(), gomock.Any()).Return(tc.testVMListWithoutInstanceView, tc.vmListErr).AnyTimes()
+		mockVMClient.EXPECT().ListVmssFlexVMsWithOnlyInstanceView(gomock.Any(), gomock.Any()).Return(tc.testVMListWithOnlyInstanceView, tc.vmListErr).AnyTimes()
+
+		mockInterfacesClient := fs.InterfacesClient.(*mockinterfaceclient.MockInterface)
+		mockInterfacesClient.EXPECT().Get(gomock.Any(), gomock.Any(), "testvm1-nic", gomock.Any()).Return(tc.nic, tc.nicGetErr).AnyTimes()
+		mockInterfacesClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(tc.nicPutErr).AnyTimes()
+
+		err = fs.EnsureBackendPoolDeleted(tc.service, tc.backendPoolID, tc.vmSetName, tc.backendAddressPools, tc.deleteFromVMSet)
+
+		if tc.expectedErr != nil {
+			assert.EqualError(t, err, tc.expectedErr.Error(), tc.description)
+		} else {
+			assert.NoError(t, err, tc.description)
+		}
+	}
+
 }