Repository: kubernetes-sigs/cloud-provider-azure
Pull request #556 (2021-03-14 08:00:27)

fix flaky test
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/release.md#issue-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. Follow the instructions for writing a release note: https://git.k8s.io/community/contributors/guide/release-notes.md
6. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

**What type of PR is this?**
> Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line:
>
> /kind api-change
> /kind bug
> /kind cleanup
> /kind design
> /kind documentation
> /kind failing-test
> /kind feature
> /kind flake
/kind failing-test

**What this PR does / why we need it**:
fix flaky test

**Which issue(s) this PR fixes**:
<!-- 
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

**Special notes for your reviewer**:


**Release note**:
```
fix flaky test
```



Patch:
diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go
index d84e966cc..0160d1d53 100644
--- a/pkg/consts/consts.go
+++ b/pkg/consts/consts.go
@@ -106,15 +106,6 @@ const (
 
 	// StrRawVersion is the raw version string
 	StrRawVersion string = "raw"
-
-	// VirtualMachineScaleSetsDeallocating indicates VMSS instances are in Deallocating state.
-	VirtualMachineScaleSetsDeallocating = "Deallocating"
-	// VmssMachineIDTemplate is the vmss manchine ID template
-	VmssMachineIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%s"
-	// VMSetCIDRIPV4TagKey specifies the node ipv4 CIDR mask of the instances on the VMSS or VMAS
-	VMSetCIDRIPV4TagKey = "kubernetesNodeCIDRMaskIPV4"
-	// VMSetCIDRIPV6TagKey specifies the node ipv6 CIDR mask of the instances on the VMSS or VMAS
-	VMSetCIDRIPV6TagKey = "kubernetesNodeCIDRMaskIPV6"
 )
 
 // azure cloud config
diff --git a/pkg/provider/azure_backoff.go b/pkg/provider/azure_backoff.go
index 8efee9510..9926fcd4a 100644
--- a/pkg/provider/azure_backoff.go
+++ b/pkg/provider/azure_backoff.go
@@ -436,7 +436,7 @@ func (az *Cloud) CreateOrUpdateVMSS(resourceGroupName string, VMScaleSetName str
 		klog.Errorf("CreateOrUpdateVMSS: error getting vmss(%s): %v", VMScaleSetName, rerr)
 		return rerr
 	}
-	if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+	if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, virtualMachineScaleSetsDeallocating) {
 		klog.V(3).Infof("CreateOrUpdateVMSS: found vmss %s being deleted, skipping", VMScaleSetName)
 		return nil
 	}
diff --git a/pkg/provider/azure_vmss.go b/pkg/provider/azure_vmss.go
index cb2ea0793..775177320 100644
--- a/pkg/provider/azure_vmss.go
+++ b/pkg/provider/azure_vmss.go
@@ -36,7 +36,6 @@ import (
 	utilnet "k8s.io/utils/net"
 
 	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
-	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 	"sigs.k8s.io/cloud-provider-azure/pkg/metrics"
 )
 
@@ -51,6 +50,18 @@ var (
 	vmssVMProviderIDRE     = regexp.MustCompile(`azure:///subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines/(?:\d+)`)
 )
 
+const (
+	// virtualMachineScaleSetsDeallocating indicates VMSS instances are in Deallocating state.
+	virtualMachineScaleSetsDeallocating = "Deallocating"
+
+	vmssMachineIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%s"
+
+	// VMSetCIDRIPV4TagKey specifies the node ipv4 CIDR mask of the instances on the VMSS or VMAS
+	VMSetCIDRIPV4TagKey = "kubernetesNodeCIDRMaskIPV4"
+	// VMSetCIDRIPV6TagKey specifies the node ipv6 CIDR mask of the instances on the VMSS or VMAS
+	VMSetCIDRIPV6TagKey = "kubernetesNodeCIDRMaskIPV6"
+)
+
 // vmssMetaInfo contains the metadata for a VMSS.
 type vmssMetaInfo struct {
 	vmssName      string
@@ -551,7 +562,7 @@ func (ss *ScaleSet) getPrimaryInterfaceID(machine compute.VirtualMachineScaleSet
 // getVmssMachineID returns the full identifier of a vmss virtual machine.
 func (az *Cloud) getVmssMachineID(subscriptionID, resourceGroup, scaleSetName, instanceID string) string {
 	return fmt.Sprintf(
-		consts.VmssMachineIDTemplate,
+		vmssMachineIDTemplate,
 		subscriptionID,
 		strings.ToLower(resourceGroup),
 		scaleSetName,
@@ -1087,7 +1098,7 @@ func (ss *ScaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, virtualMachineScaleSetsDeallocating) {
 			klog.V(3).Infof("ensureVMSSInPool: found vmss %s being deleted, skipping", vmssName)
 			continue
 		}
@@ -1436,7 +1447,7 @@ func (ss *ScaleSet) ensureBackendPoolDeletedFromVMSS(service *v1.Service, backen
 
 		// When vmss is being deleted, CreateOrUpdate API would report "the vmss is being deleted" error.
 		// Since it is being deleted, we shouldn't send more CreateOrUpdate requests for it.
-		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, consts.VirtualMachineScaleSetsDeallocating) {
+		if vmss.ProvisioningState != nil && strings.EqualFold(*vmss.ProvisioningState, virtualMachineScaleSetsDeallocating) {
 			klog.V(3).Infof("ensureVMSSInPool: found vmss %s being deleted, skipping", vmssName)
 			continue
 		}
@@ -1644,13 +1655,13 @@ func (ss *ScaleSet) GetNodeCIDRMasksByProviderID(providerID string) (int, int, e
 	}
 
 	var ipv4Mask, ipv6Mask int
-	if v4, ok := vmss.Tags[consts.VMSetCIDRIPV4TagKey]; ok && v4 != nil {
+	if v4, ok := vmss.Tags[VMSetCIDRIPV4TagKey]; ok && v4 != nil {
 		ipv4Mask, err = strconv.Atoi(to.String(v4))
 		if err != nil {
 			klog.Errorf(" GetNodeCIDRMasksByProviderID: error when paring the value of the ipv4 mask size %s: %v", to.String(v4), err)
 		}
 	}
-	if v6, ok := vmss.Tags[consts.VMSetCIDRIPV6TagKey]; ok && v6 != nil {
+	if v6, ok := vmss.Tags[VMSetCIDRIPV6TagKey]; ok && v6 != nil {
 		ipv6Mask, err = strconv.Atoi(to.String(v6))
 		if err != nil {
 			klog.Errorf(" GetNodeCIDRMasksByProviderID: error when paring the value of the ipv6 mask size%s: %v", to.String(v6), err)


Test patch:
diff --git a/pkg/nodeipam/ipam/cloud_cidr_allocator_test.go b/pkg/nodeipam/ipam/cloud_cidr_allocator_test.go
index a9c2acf0c..4a5b07374 100644
--- a/pkg/nodeipam/ipam/cloud_cidr_allocator_test.go
+++ b/pkg/nodeipam/ipam/cloud_cidr_allocator_test.go
@@ -33,7 +33,6 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 
 	"sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssclient/mockvmssclient"
-	"sigs.k8s.io/cloud-provider-azure/pkg/consts"
 	azureprovider "sigs.k8s.io/cloud-provider-azure/pkg/provider"
 	"sigs.k8s.io/cloud-provider-azure/pkg/util/controller/testutil"
 )
@@ -190,8 +189,8 @@ func TestUpdateNodeSubnetMaskSizes(t *testing.T) {
 			description: "updateNodeSubnetMaskSizes should put the correct mask sizes on the map",
 			providerID:  "azure:///subscriptions/sub/resourceGroups/rg1/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
 			tags: map[string]*string{
-				consts.VMSetCIDRIPV4TagKey: to.StringPtr("25"),
-				consts.VMSetCIDRIPV6TagKey: to.StringPtr("65"),
+				azureprovider.VMSetCIDRIPV4TagKey: to.StringPtr("25"),
+				azureprovider.VMSetCIDRIPV6TagKey: to.StringPtr("65"),
 			},
 			expectedNodeNameSubnetMaskSizesMap: map[string][]int{"vmss-0": {25, 65}},
 		},
@@ -199,8 +198,8 @@ func TestUpdateNodeSubnetMaskSizes(t *testing.T) {
 			description: "updateNodeSubnetMaskSizes should put the default mask sizes on the map if the providerID is invalid",
 			providerID:  "invalid",
 			tags: map[string]*string{
-				consts.VMSetCIDRIPV4TagKey: to.StringPtr("24"),
-				consts.VMSetCIDRIPV6TagKey: to.StringPtr("64"),
+				azureprovider.VMSetCIDRIPV4TagKey: to.StringPtr("24"),
+				azureprovider.VMSetCIDRIPV6TagKey: to.StringPtr("64"),
 			},
 			expectedNodeNameSubnetMaskSizesMap: map[string][]int{"vmss-0": {24, 64}},
 		},
@@ -208,8 +207,8 @@ func TestUpdateNodeSubnetMaskSizes(t *testing.T) {
 			description: "updateNodeSubnetMaskSizes should report an error if the ipv4 mask is smaller than the cluster mask",
 			providerID:  "azure:///subscriptions/sub/resourceGroups/rg1/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
 			tags: map[string]*string{
-				consts.VMSetCIDRIPV4TagKey: to.StringPtr("15"),
-				consts.VMSetCIDRIPV6TagKey: to.StringPtr("65"),
+				azureprovider.VMSetCIDRIPV4TagKey: to.StringPtr("15"),
+				azureprovider.VMSetCIDRIPV6TagKey: to.StringPtr("65"),
 			},
 			expectedNodeNameSubnetMaskSizesMap: map[string][]int{},
 			expectedErr:                        fmt.Errorf("updateNodeSubnetMaskSizes: invalid ipv4 mask size %d of node %s because it is out of the range of the cluster CIDR with the mask size %d", 15, "vmss-0", 16),
@@ -218,8 +217,8 @@ func TestUpdateNodeSubnetMaskSizes(t *testing.T) {
 			description: "updateNodeSubnetMaskSizes should report an error if the ipv6 mask is smaller than the cluster mask",
 			providerID:  "azure:///subscriptions/sub/resourceGroups/rg1/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
 			tags: map[string]*string{
-				consts.VMSetCIDRIPV4TagKey: to.StringPtr("25"),
-				consts.VMSetCIDRIPV6TagKey: to.StringPtr("45"),
+				azureprovider.VMSetCIDRIPV4TagKey: to.StringPtr("25"),
+				azureprovider.VMSetCIDRIPV6TagKey: to.StringPtr("45"),
 			},
 			expectedNodeNameSubnetMaskSizesMap: map[string][]int{},
 			expectedErr:                        fmt.Errorf("updateNodeSubnetMaskSizes: invalid ipv6 mask size %d of node %s because it is out of the range of the cluster CIDR with the mask size %d", 45, "vmss-0", 48),
diff --git a/pkg/provider/azure_backoff_test.go b/pkg/provider/azure_backoff_test.go
index 00d8b0af3..61a68f24d 100644
--- a/pkg/provider/azure_backoff_test.go
+++ b/pkg/provider/azure_backoff_test.go
@@ -506,7 +506,7 @@ func TestCreateOrUpdateVMSS(t *testing.T) {
 		{
 			vmss: compute.VirtualMachineScaleSet{
 				VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
-					ProvisioningState: to.StringPtr(consts.VirtualMachineScaleSetsDeallocating),
+					ProvisioningState: to.StringPtr(virtualMachineScaleSetsDeallocating),
 				},
 			},
 		},
diff --git a/pkg/provider/azure_vmss_test.go b/pkg/provider/azure_vmss_test.go
index 0334cfa8f..b19b9775c 100644
--- a/pkg/provider/azure_vmss_test.go
+++ b/pkg/provider/azure_vmss_test.go
@@ -1207,7 +1207,7 @@ func TestGetVmssMachineID(t *testing.T) {
 
 	subscriptionID, resourceGroup, scaleSetName, instanceID := "sub", "RG", "vmss", "id"
 	VMSSMachineID := ss.cloud.getVmssMachineID(subscriptionID, resourceGroup, scaleSetName, instanceID)
-	expectedVMSSMachineID := fmt.Sprintf(consts.VmssMachineIDTemplate, subscriptionID, strings.ToLower(resourceGroup), scaleSetName, instanceID)
+	expectedVMSSMachineID := fmt.Sprintf(vmssMachineIDTemplate, subscriptionID, strings.ToLower(resourceGroup), scaleSetName, instanceID)
 	assert.Equal(t, expectedVMSSMachineID, VMSSMachineID, "GetVmssMachineID should return the correct VMSS machine ID")
 }
 
@@ -1977,7 +1977,7 @@ func TestEnsureVMSSInPool(t *testing.T) {
 
 		expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, test.setIPv6Config)
 		if test.isVMSSDeallocating {
-			expectedVMSS.ProvisioningState = to.StringPtr(consts.VirtualMachineScaleSetsDeallocating)
+			expectedVMSS.ProvisioningState = to.StringPtr(virtualMachineScaleSetsDeallocating)
 		}
 		if test.isVMSSNilNICConfig {
 			expectedVMSS.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = nil
@@ -2251,7 +2251,7 @@ func TestEnsureBackendPoolDeletedFromVMSS(t *testing.T) {
 
 		expectedVMSS := buildTestVMSSWithLB(testVMSSName, "vmss-vm-", []string{testLBBackendpoolID0}, false)
 		if test.isVMSSDeallocating {
-			expectedVMSS.ProvisioningState = to.StringPtr(consts.VirtualMachineScaleSetsDeallocating)
+			expectedVMSS.ProvisioningState = to.StringPtr(virtualMachineScaleSetsDeallocating)
 		}
 		if test.isVMSSNilNICConfig {
 			expectedVMSS.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = nil
@@ -2464,8 +2464,8 @@ func TestGetNodeCIDRMasksByProviderID(t *testing.T) {
 			description: "GetNodeCIDRMaksByProviderID should return the correct mask sizes",
 			providerID:  "azure:///subscriptions/sub/resourceGroups/rg1/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
 			tags: map[string]*string{
-				consts.VMSetCIDRIPV4TagKey: to.StringPtr("24"),
-				consts.VMSetCIDRIPV6TagKey: to.StringPtr("64"),
+				VMSetCIDRIPV4TagKey: to.StringPtr("24"),
+				VMSetCIDRIPV6TagKey: to.StringPtr("64"),
 			},
 			expectedIPV4MaskSize: 24,
 			expectedIPV6MaskSize: 64,
@@ -2474,7 +2474,7 @@ func TestGetNodeCIDRMasksByProviderID(t *testing.T) {
 			description: "GetNodeCIDRMaksByProviderID should return the correct mask sizes even if some of the tags are not specified",
 			providerID:  "azure:///subscriptions/sub/resourceGroups/rg1/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
 			tags: map[string]*string{
-				consts.VMSetCIDRIPV4TagKey: to.StringPtr("24"),
+				VMSetCIDRIPV4TagKey: to.StringPtr("24"),
 			},
 			expectedIPV4MaskSize: 24,
 		},
@@ -2482,8 +2482,8 @@ func TestGetNodeCIDRMasksByProviderID(t *testing.T) {
 			description: "GetNodeCIDRMaksByProviderID should fail even if some of the tag is invalid",
 			providerID:  "azure:///subscriptions/sub/resourceGroups/rg1/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0",
 			tags: map[string]*string{
-				consts.VMSetCIDRIPV4TagKey: to.StringPtr("abc"),
-				consts.VMSetCIDRIPV6TagKey: to.StringPtr("64"),
+				VMSetCIDRIPV4TagKey: to.StringPtr("abc"),
+				VMSetCIDRIPV6TagKey: to.StringPtr("64"),
 			},
 			expectedIPV6MaskSize: 64,
 		},
diff --git a/tests/e2e/network/standard_lb.go b/tests/e2e/network/standard_lb.go
index 8aa80adeb..205cc3924 100644
--- a/tests/e2e/network/standard_lb.go
+++ b/tests/e2e/network/standard_lb.go
@@ -20,6 +20,8 @@ import (
 	"context"
 	"strings"
 
+	"github.com/Azure/go-autorest/autorest/to"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -95,6 +97,7 @@ var _ = Describe("[StandardLoadBalancer] Standard load balancer", func() {
 
 		lbBackendAddressPoolsIDMap := make(map[string]bool)
 		for _, backendAddressPool := range *lb.BackendAddressPools {
+			utils.Logf("found backend pool %s", to.String(backendAddressPool.ID))
 			lbBackendAddressPoolsIDMap[*backendAddressPool.ID] = true
 		}
 
@@ -106,9 +109,15 @@ var _ = Describe("[StandardLoadBalancer] Standard load balancer", func() {
 				continue
 			}
 			for _, ipConfig := range *nic.IPConfigurations {
+				if found {
+					break
+				}
+				utils.Logf("found ip config %s", to.String(ipConfig.Name))
 				for _, backendAddressPool := range *ipConfig.LoadBalancerBackendAddressPools {
+					utils.Logf("found backend pool on nic %s", to.String(backendAddressPool.ID))
 					if lbBackendAddressPoolsIDMap[*backendAddressPool.ID] {
 						found = true
+						break
 					}
 				}
 			}