Repository: kubernetes-sigs/cloud-provider-azure
Pull request #3889 (2023-05-17 13:14:30)

[release-1.27] fix: the pip without tags should be user-assigned
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?

<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

/kind bug

#### What this PR does / why we need it:

1. PIPs without tags should be treated as user-assigned resources. We should not tag user-assigned pip without tags.
2. When the svc changes its IP from one to another, the cached pip can be outdated. Need to refresh the cache when necessary.
3. We should not tag user-assigned pip for DNS labels.

#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
fix: the pip without tags should be user-assigned
fix: refresh the pip cache when necessary
fix: do not tag user-assigned pip with `kubernetes-dns-label-service: <svcName>`
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```



Patch:
diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go
index 9386bb4b9..f015968a3 100644
--- a/pkg/provider/azure_loadbalancer.go
+++ b/pkg/provider/azure_loadbalancer.go
@@ -897,19 +897,36 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service,
 }
 
 func (az *Cloud) findMatchedPIPByLoadBalancerIP(service *v1.Service, loadBalancerIP, pipResourceGroup string) (*network.PublicIPAddress, error) {
-	pips, err := az.listPIP(pipResourceGroup)
+	pips, err := az.listPIP(pipResourceGroup, azcache.CacheReadTypeDefault)
 	if err != nil {
 		return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: failed to listPIP: %w", err)
 	}
+
+	pip, err := getExpectedPIPFromListByIPAddress(pips, loadBalancerIP)
+	if err != nil {
+		pips, err = az.listPIP(pipResourceGroup, azcache.CacheReadTypeForceRefresh)
+		if err != nil {
+			return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: failed to listPIP force refresh: %w", err)
+		}
+
+		pip, err = getExpectedPIPFromListByIPAddress(pips, loadBalancerIP)
+		if err != nil {
+			return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: cannot find public IP with IP address %s in resource group %s", loadBalancerIP, pipResourceGroup)
+		}
+	}
+
+	return pip, nil
+}
+
+func getExpectedPIPFromListByIPAddress(pips []network.PublicIPAddress, ip string) (*network.PublicIPAddress, error) {
 	for _, pip := range pips {
-		pip := pip
 		if pip.PublicIPAddressPropertiesFormat.IPAddress != nil &&
-			*pip.PublicIPAddressPropertiesFormat.IPAddress == loadBalancerIP {
+			*pip.PublicIPAddressPropertiesFormat.IPAddress == ip {
 			return &pip, nil
 		}
 	}
 
-	return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: cannot find public IP with IP address %s in resource group %s", loadBalancerIP, pipResourceGroup)
+	return nil, fmt.Errorf("getExpectedPIPFromListByIPAddress: cannot find public IP with IP address %s", ip)
 }
 
 func flipServiceInternalAnnotation(service *v1.Service) *v1.Service {
@@ -1080,7 +1097,7 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai
 	}
 
 	if foundDNSLabelAnnotation {
-		updatedDNSSettings, err := reconcileDNSSettings(&pip, domainNameLabel, serviceName, pipName)
+		updatedDNSSettings, err := reconcileDNSSettings(&pip, domainNameLabel, serviceName, pipName, isUserAssignedPIP)
 		if err != nil {
 			return nil, fmt.Errorf("ensurePublicIPExists for service(%s): failed to reconcileDNSSettings: %w", serviceName, err)
 		}
@@ -1149,7 +1166,11 @@ func (az *Cloud) reconcileIPSettings(pip *network.PublicIPAddress, service *v1.S
 	return changed
 }
 
-func reconcileDNSSettings(pip *network.PublicIPAddress, domainNameLabel, serviceName, pipName string) (bool, error) {
+func reconcileDNSSettings(
+	pip *network.PublicIPAddress,
+	domainNameLabel, serviceName, pipName string,
+	isUserAssignedPIP bool,
+) (bool, error) {
 	var changed bool
 
 	if existingServiceName := getServiceFromPIPDNSTags(pip.Tags); existingServiceName != "" && !strings.EqualFold(existingServiceName, serviceName) {
@@ -1178,8 +1199,10 @@ func reconcileDNSSettings(pip *network.PublicIPAddress, domainNameLabel, service
 		}
 
 		if svc := getServiceFromPIPDNSTags(pip.Tags); svc == "" || !strings.EqualFold(svc, serviceName) {
-			pip.Tags[consts.ServiceUsingDNSKey] = &serviceName
-			changed = true
+			if !isUserAssignedPIP {
+				pip.Tags[consts.ServiceUsingDNSKey] = &serviceName
+				changed = true
+			}
 		}
 	}
 
@@ -3016,7 +3039,7 @@ func (az *Cloud) reconcilePublicIPs(clusterName string, service *v1.Service, lbN
 	pipResourceGroup := az.getPublicIPAddressResourceGroup(service)
 
 	reconciledPIPs := []*network.PublicIPAddress{}
-	pips, err := az.listPIP(pipResourceGroup)
+	pips, err := az.listPIP(pipResourceGroup, azcache.CacheReadTypeDefault)
 	if err != nil {
 		return nil, err
 	}
@@ -3165,7 +3188,7 @@ func (az *Cloud) getPublicIPUpdates(
 		owns, isUserAssignedPIP := serviceOwnsPublicIP(service, &pip, clusterName)
 		if owns {
 			var dirtyPIP, toBeDeleted bool
-			if !wantLb {
+			if !wantLb && !isUserAssignedPIP {
 				klog.V(2).Infof("reconcilePublicIP for service(%s): unbinding the service from pip %s", serviceName, *pip.Name)
 				if err = unbindServiceFromPIP(&pip, service, serviceName, clusterName, isUserAssignedPIP); err != nil {
 					return false, nil, false, nil, err
@@ -3210,65 +3233,77 @@ func (az *Cloud) getPublicIPUpdates(
 func (az *Cloud) safeDeletePublicIP(service *v1.Service, pipResourceGroup string, pip *network.PublicIPAddress, lb *network.LoadBalancer) error {
 	// Remove references if pip.IPConfiguration is not nil.
 	if pip.PublicIPAddressPropertiesFormat != nil &&
-		pip.PublicIPAddressPropertiesFormat.IPConfiguration != nil &&
-		lb != nil && lb.LoadBalancerPropertiesFormat != nil &&
-		lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations != nil {
-		referencedLBRules := []network.SubResource{}
-		frontendIPConfigUpdated := false
-		loadBalancerRuleUpdated := false
-
-		// Check whether there are still frontend IP configurations referring to it.
-		ipConfigurationID := pointer.StringDeref(pip.PublicIPAddressPropertiesFormat.IPConfiguration.ID, "")
-		if ipConfigurationID != "" {
-			lbFrontendIPConfigs := *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations
-			for i := len(lbFrontendIPConfigs) - 1; i >= 0; i-- {
-				config := lbFrontendIPConfigs[i]
-				if strings.EqualFold(ipConfigurationID, pointer.StringDeref(config.ID, "")) {
-					if config.FrontendIPConfigurationPropertiesFormat != nil &&
-						config.FrontendIPConfigurationPropertiesFormat.LoadBalancingRules != nil {
-						referencedLBRules = *config.FrontendIPConfigurationPropertiesFormat.LoadBalancingRules
-					}
+		pip.PublicIPAddressPropertiesFormat.IPConfiguration != nil {
+		// Fetch latest pip to check if the pip in the cache is stale.
+		// In some cases the public IP to be deleted is still referencing
+		// the frontend IP config on the LB. This is because the pip is
+		// stored in the cache and is not up-to-date.
+		latestPIP, ok, err := az.getPublicIPAddress(pipResourceGroup, *pip.Name, azcache.CacheReadTypeForceRefresh)
+		if err != nil {
+			klog.Errorf("safeDeletePublicIP: failed to get latest public IP %s/%s: %s", pipResourceGroup, *pip.Name, err.Error())
+			return err
+		}
+		if ok && latestPIP.PublicIPAddressPropertiesFormat != nil &&
+			latestPIP.PublicIPAddressPropertiesFormat.IPConfiguration != nil &&
+			lb != nil && lb.LoadBalancerPropertiesFormat != nil &&
+			lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations != nil {
+			referencedLBRules := []network.SubResource{}
+			frontendIPConfigUpdated := false
+			loadBalancerRuleUpdated := false
+
+			// Check whether there are still frontend IP configurations referring to it.
+			ipConfigurationID := pointer.StringDeref(pip.PublicIPAddressPropertiesFormat.IPConfiguration.ID, "")
+			if ipConfigurationID != "" {
+				lbFrontendIPConfigs := *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations
+				for i := len(lbFrontendIPConfigs) - 1; i >= 0; i-- {
+					config := lbFrontendIPConfigs[i]
+					if strings.EqualFold(ipConfigurationID, pointer.StringDeref(config.ID, "")) {
+						if config.FrontendIPConfigurationPropertiesFormat != nil &&
+							config.FrontendIPConfigurationPropertiesFormat.LoadBalancingRules != nil {
+							referencedLBRules = *config.FrontendIPConfigurationPropertiesFormat.LoadBalancingRules
+						}
 
-					frontendIPConfigUpdated = true
-					lbFrontendIPConfigs = append(lbFrontendIPConfigs[:i], lbFrontendIPConfigs[i+1:]...)
-					break
+						frontendIPConfigUpdated = true
+						lbFrontendIPConfigs = append(lbFrontendIPConfigs[:i], lbFrontendIPConfigs[i+1:]...)
+						break
+					}
 				}
-			}
 
-			if frontendIPConfigUpdated {
-				lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = &lbFrontendIPConfigs
+				if frontendIPConfigUpdated {
+					lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations = &lbFrontendIPConfigs
+				}
 			}
-		}
 
-		// Check whether there are still load balancer rules referring to it.
-		if len(referencedLBRules) > 0 {
-			referencedLBRuleIDs := sets.New[string]()
-			for _, refer := range referencedLBRules {
-				referencedLBRuleIDs.Insert(pointer.StringDeref(refer.ID, ""))
-			}
+			// Check whether there are still load balancer rules referring to it.
+			if len(referencedLBRules) > 0 {
+				referencedLBRuleIDs := sets.New[string]()
+				for _, refer := range referencedLBRules {
+					referencedLBRuleIDs.Insert(pointer.StringDeref(refer.ID, ""))
+				}
 
-			if lb.LoadBalancerPropertiesFormat.LoadBalancingRules != nil {
-				lbRules := *lb.LoadBalancerPropertiesFormat.LoadBalancingRules
-				for i := len(lbRules) - 1; i >= 0; i-- {
-					ruleID := pointer.StringDeref(lbRules[i].ID, "")
-					if ruleID != "" && referencedLBRuleIDs.Has(ruleID) {
-						loadBalancerRuleUpdated = true
-						lbRules = append(lbRules[:i], lbRules[i+1:]...)
+				if lb.LoadBalancerPropertiesFormat.LoadBalancingRules != nil {
+					lbRules := *lb.LoadBalancerPropertiesFormat.LoadBalancingRules
+					for i := len(lbRules) - 1; i >= 0; i-- {
+						ruleID := pointer.StringDeref(lbRules[i].ID, "")
+						if ruleID != "" && referencedLBRuleIDs.Has(ruleID) {
+							loadBalancerRuleUpdated = true
+							lbRules = append(lbRules[:i], lbRules[i+1:]...)
+						}
 					}
-				}
 
-				if loadBalancerRuleUpdated {
-					lb.LoadBalancerPropertiesFormat.LoadBalancingRules = &lbRules
+					if loadBalancerRuleUpdated {
+						lb.LoadBalancerPropertiesFormat.LoadBalancingRules = &lbRules
+					}
 				}
 			}
-		}
 
-		// Update load balancer when frontendIPConfigUpdated or loadBalancerRuleUpdated.
-		if frontendIPConfigUpdated || loadBalancerRuleUpdated {
-			err := az.CreateOrUpdateLB(service, *lb)
-			if err != nil {
-				klog.Errorf("safeDeletePublicIP for service(%s) failed with error: %v", getServiceName(service), err)
-				return err
+			// Update load balancer when frontendIPConfigUpdated or loadBalancerRuleUpdated.
+			if frontendIPConfigUpdated || loadBalancerRuleUpdated {
+				err := az.CreateOrUpdateLB(service, *lb)
+				if err != nil {
+					klog.Errorf("safeDeletePublicIP for service(%s) failed with error: %v", getServiceName(service), err)
+					return err
+				}
 			}
 		}
 	}
@@ -3527,13 +3562,14 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus
 
 	serviceName := getServiceName(service)
 
+	isIPv6 := pip.PublicIPAddressVersion == network.IPv6
 	if pip.Tags != nil {
 		serviceTag := getServiceFromPIPServiceTags(pip.Tags)
 		clusterTag := getClusterFromPIPClusterTags(pip.Tags)
 
 		// if there is no service tag on the pip, it is user-created pip
 		if serviceTag == "" {
-			return strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), getServiceLoadBalancerIP(service, utilnet.IsIPv6String(service.Spec.ClusterIP))), true
+			return isServiceLoadBalancerIPMatchesPIP(service, pip, isIPv6), true
 		}
 
 		// if there is service tag on the pip, it is system-created pip
@@ -3545,17 +3581,20 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus
 			}
 
 			// If cluster name tag is set, then return true if it matches.
-			if clusterTag == clusterName {
-				return true, false
-			}
-		} else {
-			// if the service is not included in the tags of the system-created pip, check the ip address
-			// this could happen for secondary services
-			return strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), getServiceLoadBalancerIP(service, utilnet.IsIPv6String(service.Spec.ClusterIP))), false
+			return strings.EqualFold(clusterTag, clusterName), false
 		}
+
+		// if the service is not included in the tags of the system-created pip, check the ip address
+		// this could happen for secondary services
+		return isServiceLoadBalancerIPMatchesPIP(service, pip, isIPv6), false
 	}
 
-	return false, false
+	// if the pip has no tags, it should be user-created
+	return isServiceLoadBalancerIPMatchesPIP(service, pip, isIPv6), true
+}
+
+func isServiceLoadBalancerIPMatchesPIP(service *v1.Service, pip *network.PublicIPAddress, isIPV6 bool) bool {
+	return strings.EqualFold(pointer.StringDeref(pip.IPAddress, ""), getServiceLoadBalancerIP(service, isIPV6))
 }
 
 func isSVCNameInPIPTag(tag, svcName string) bool {
diff --git a/pkg/provider/azure_utils.go b/pkg/provider/azure_utils.go
index 4fdc8d4b2..9dcb2879c 100644
--- a/pkg/provider/azure_utils.go
+++ b/pkg/provider/azure_utils.go
@@ -448,7 +448,7 @@ func getResourceByIPFamily(resource string, isDualStack, isIPv6 bool) string {
 
 // isFIPIPv6 checks if the frontend IP configuration is of IPv6.
 func (az *Cloud) isFIPIPv6(fip *network.FrontendIPConfiguration, pipResourceGroup string, isInternal bool) (isIPv6 bool, err error) {
-	pips, err := az.listPIP(pipResourceGroup)
+	pips, err := az.listPIP(pipResourceGroup, azcache.CacheReadTypeDefault)
 	if err != nil {
 		return false, fmt.Errorf("isFIPIPv6: failed to list pip: %w", err)
 	}
diff --git a/pkg/provider/azure_wrap.go b/pkg/provider/azure_wrap.go
index 6a883ba57..3a4277ab6 100644
--- a/pkg/provider/azure_wrap.go
+++ b/pkg/provider/azure_wrap.go
@@ -125,8 +125,8 @@ func (az *Cloud) getPublicIPAddress(pipResourceGroup string, pipName string, crt
 	return *(deepcopy.Copy(pip).(*network.PublicIPAddress)), true, nil
 }
 
-func (az *Cloud) listPIP(pipResourceGroup string) ([]network.PublicIPAddress, error) {
-	cached, err := az.pipCache.Get(pipResourceGroup, azcache.CacheReadTypeDefault)
+func (az *Cloud) listPIP(pipResourceGroup string, crt azcache.AzureCacheReadType) ([]network.PublicIPAddress, error) {
+	cached, err := az.pipCache.Get(pipResourceGroup, crt)
 	if err != nil {
 		return nil, err
 	}
diff --git a/site/content/en/topics/shared-ip.md b/site/content/en/topics/shared-ip.md
index ab5fb264c..8e61eaf26 100644
--- a/site/content/en/topics/shared-ip.md
+++ b/site/content/en/topics/shared-ip.md
@@ -35,7 +35,8 @@ kind: Service
 metadata:
   name: https
   namespace: default
-  service.beta.kubernetes.io/azure-load-balancer-ipv4: 1.2.3.4 # the IP address could be the same as it is of `nginx` service
+  annotations:
+    service.beta.kubernetes.io/azure-load-balancer-ipv4: 1.2.3.4 # the IP address could be the same as it is of `nginx` service
 spec:
   ports:
     - port: 443
@@ -50,9 +51,9 @@ Note that if you specify the annotations `service.beta.kubernetes.io/azure-load-
 
 ## DNS
 
-Even if multiple services can refer to one public IP, the DNS label cannot be re-used. The public IP would have the label `kubernetes-dns-label-service: <svcName>` to indicate which service is binding to the DNS label. In this case if there is another service sharing this specific IP address trying to refer to the DNS label, an error would be reported.
+Even if multiple services can refer to one public IP, the DNS label cannot be re-used. The public IP would have the label `kubernetes-dns-label-service: <svcName>` to indicate which service is binding to the DNS label. In this case if there is another service sharing this specific IP address trying to refer to the DNS label, an error would be reported. For managed public IPs, this label will be added automatically by the cloud provider. For static public IPs, this label should be added manually.
 
-> The DNS name on the public IP won't be deleted after the service with the DNS annotation being deleted, because the cloud provider don't know if the DNS was set by the user or not.
+```yaml
 
 ## Restrictions
 


Test patch:
diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go
index 8bb54109f..211ef9c0c 100644
--- a/pkg/provider/azure_loadbalancer_test.go
+++ b/pkg/provider/azure_loadbalancer_test.go
@@ -1081,6 +1081,17 @@ func TestServiceOwnsPublicIP(t *testing.T) {
 			serviceLBIP:  "1.1.1.1",
 			expectedOwns: true,
 		},
+		{
+			desc: "should be user-assigned pip if it has no tags",
+			pip: &network.PublicIPAddress{
+				PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
+					IPAddress: pointer.String("1.2.3.4"),
+				},
+			},
+			serviceLBIP:             "1.2.3.4",
+			expectedOwns:            true,
+			expectedUserAssignedPIP: true,
+		},
 	}
 
 	for i, c := range tests {
@@ -2204,10 +2215,11 @@ func TestFindMatchedPIPByLoadBalancerIP(t *testing.T) {
 		},
 	}
 	testCases := []struct {
-		desc          string
-		pips          []network.PublicIPAddress
-		expectedPIP   *network.PublicIPAddress
-		expectedError bool
+		desc               string
+		pips               []network.PublicIPAddress
+		shouldRefreshCache bool
+		expectedPIP        *network.PublicIPAddress
+		expectedError      bool
 	}{
 		{
 			desc:        "findMatchedPIPByLoadBalancerIP shall return the matched ip",
@@ -2215,9 +2227,16 @@ func TestFindMatchedPIPByLoadBalancerIP(t *testing.T) {
 			expectedPIP: &testPIP,
 		},
 		{
-			desc:          "findMatchedPIPByLoadBalancerIP shall return error if ip is not found",
-			pips:          []network.PublicIPAddress{},
-			expectedError: true,
+			desc:               "findMatchedPIPByLoadBalancerIP shall return error if ip is not found",
+			pips:               []network.PublicIPAddress{},
+			shouldRefreshCache: true,
+			expectedError:      true,
+		},
+		{
+			desc:               "findMatchedPIPByLoadBalancerIP should refresh cache if no matched ip is found",
+			pips:               []network.PublicIPAddress{testPIP},
+			shouldRefreshCache: true,
+			expectedPIP:        &testPIP,
 		},
 	}
 	for _, test := range testCases {
@@ -2227,6 +2246,9 @@ func TestFindMatchedPIPByLoadBalancerIP(t *testing.T) {
 			setServiceLoadBalancerIP(&service, "1.2.3.4")
 
 			mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface)
+			if test.shouldRefreshCache {
+				mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return([]network.PublicIPAddress{}, nil)
+			}
 			mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return(test.pips, nil)
 			pip, err := az.findMatchedPIPByLoadBalancerIP(&service, "1.2.3.4", "rg")
 			assert.Equal(t, test.expectedPIP, pip)
@@ -2283,7 +2305,7 @@ func TestDeterminePublicIPName(t *testing.T) {
 			setServiceLoadBalancerIP(&service, test.loadBalancerIP)
 
 			mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface)
-			mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return(test.existingPIPs, nil).MaxTimes(1)
+			mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return(test.existingPIPs, nil).MaxTimes(2)
 			mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 			for _, existingPIP := range test.existingPIPs {
 				mockPIPsClient.EXPECT().Get(gomock.Any(), "rg", *existingPIP.Name, gomock.Any()).Return(existingPIP, nil).AnyTimes()
@@ -4007,7 +4029,8 @@ func TestSafeDeletePublicIP(t *testing.T) {
 		desc          string
 		pip           *network.PublicIPAddress
 		lb            *network.LoadBalancer
-		expectedError bool
+		listError     *retry.Error
+		expectedError error
 	}{
 		{
 			desc: "safeDeletePublicIP shall delete corresponding ip configurations and lb rules",
@@ -4034,12 +4057,30 @@ func TestSafeDeletePublicIP(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc: "safeDeletePublicIP should return error if failed to list pip",
+			pip: &network.PublicIPAddress{
+				Name: pointer.String("pip1"),
+				PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
+					IPConfiguration: &network.IPConfiguration{
+						ID: pointer.String("id1"),
+					},
+				},
+			},
+			listError:     retry.NewError(false, errors.New("error")),
+			expectedError: retry.NewError(false, errors.New("error")).Error(),
+		},
 	}
 
 	for _, test := range testCases {
 		t.Run(test.desc, func(t *testing.T) {
 			az := GetTestCloud(ctrl)
 			mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface)
+			if test.pip != nil &&
+				test.pip.PublicIPAddressPropertiesFormat != nil &&
+				test.pip.IPConfiguration != nil {
+				mockPIPsClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]network.PublicIPAddress{*test.pip}, test.listError)
+			}
 			mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", "pip1", gomock.Any()).Return(nil).AnyTimes()
 			mockPIPsClient.EXPECT().Delete(gomock.Any(), "rg", "pip1").Return(nil).AnyTimes()
 			err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", "pip1", network.PublicIPAddress{
@@ -4052,13 +4093,19 @@ func TestSafeDeletePublicIP(t *testing.T) {
 			})
 			assert.NoError(t, err.Error())
 			service := getTestService("test1", v1.ProtocolTCP, nil, false, 80)
-			mockLBsClient := mockloadbalancerclient.NewMockInterface(ctrl)
-			mockLBsClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
-			az.LoadBalancerClient = mockLBsClient
+			if test.listError == nil {
+				mockLBsClient := mockloadbalancerclient.NewMockInterface(ctrl)
+				mockLBsClient.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)
+				az.LoadBalancerClient = mockLBsClient
+			}
 			rerr := az.safeDeletePublicIP(&service, "rg", test.pip, test.lb)
-			assert.Equal(t, 0, len(*test.lb.FrontendIPConfigurations))
-			assert.Equal(t, 0, len(*test.lb.LoadBalancingRules))
-			assert.Equal(t, test.expectedError, rerr != nil)
+			if test.expectedError == nil {
+				assert.Equal(t, 0, len(*test.lb.FrontendIPConfigurations))
+				assert.Equal(t, 0, len(*test.lb.LoadBalancingRules))
+				assert.NoError(t, rerr)
+			} else {
+				assert.Equal(t, rerr.Error(), test.listError.Error().Error())
+			}
 		})
 	}
 }
diff --git a/pkg/provider/azure_standard_test.go b/pkg/provider/azure_standard_test.go
index 97a00032c..259540e27 100644
--- a/pkg/provider/azure_standard_test.go
+++ b/pkg/provider/azure_standard_test.go
@@ -1848,7 +1848,7 @@ func TestServiceOwnsFrontendIP(t *testing.T) {
 			cloud := GetTestCloud(ctrl)
 			if test.existingPIPs != nil {
 				mockPIPsClient := cloud.PublicIPAddressesClient.(*mockpublicipclient.MockInterface)
-				mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return(test.existingPIPs, nil)
+				mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return(test.existingPIPs, nil).MaxTimes(2)
 			}
 			isOwned, isPrimary, err := cloud.serviceOwnsFrontendIP(test.fip, test.service)
 			assert.Equal(t, test.expectedErr, err)
diff --git a/pkg/provider/azure_wrap_test.go b/pkg/provider/azure_wrap_test.go
index d5bb2162e..4caee5195 100644
--- a/pkg/provider/azure_wrap_test.go
+++ b/pkg/provider/azure_wrap_test.go
@@ -396,7 +396,7 @@ func TestListPIP(t *testing.T) {
 			if test.expectPIPList {
 				mockPIPsClient.EXPECT().List(gomock.Any(), az.ResourceGroup).Return(test.existingPIPs, nil).MaxTimes(2)
 			}
-			pips, err := az.listPIP(az.ResourceGroup)
+			pips, err := az.listPIP(az.ResourceGroup, azcache.CacheReadTypeDefault)
 			if test.expectPIPList {
 				assert.ElementsMatch(t, test.existingPIPs, pips)
 			} else {
diff --git a/tests/e2e/network/service_annotations.go b/tests/e2e/network/service_annotations.go
index 60052b2ef..d91753282 100644
--- a/tests/e2e/network/service_annotations.go
+++ b/tests/e2e/network/service_annotations.go
@@ -218,7 +218,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
 		err = utils.DeleteService(cs, ns.Name, oldServiceName)
 		Expect(err).NotTo(HaveOccurred())
 
-		By("Check if PIP DNS label is deleted")
+		By("Check if PIP DNS label is not tagged onto the user-assigned pip")
 		for _, pipName := range pipNames {
 			deleted, err := ifPIPDNSLabelDeleted(tc, pipName)
 			Expect(err).NotTo(HaveOccurred())