Repository: kubernetes-sigs/cloud-provider-azure
Pull request #2470 (2022-10-11 03:22:04)

[release-1.1] Deprecate LoadBalancerIP with Servie LB IP annotation
* Add service.beta.kubernetes.io/azure-load-balancer-ipv4 and service.beta.kubernetes.io/azure-load-balancer-ipv6
* Add an e2e test
* Support retrieve Service CIDR in e2e framework

Signed-off-by: Zhecheng Li <zhechengli@microsoft.com>

<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?
/kind cleanup
/kind feature
/kind deprecation
<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

#### What this PR does / why we need it:
Deprecate LoadBalancerIP with Servie LB IP annotation
#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
Deprecate LoadBalancerIP with Servie LB IP annotation
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```


k8s-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubernetes-sigs/cloud-provider-azure/pull/2470#" title="Author self-approved">lzhecheng</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fcloud-provider-azure).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubernetes-sigs/cloud-provider-azure/blob/release-1.1/OWNERS)~~ [lzhecheng]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go
index 3e5d88459..2f2acd931 100644
--- a/pkg/consts/consts.go
+++ b/pkg/consts/consts.go
@@ -172,6 +172,16 @@ const (
 	BackoffJitterDefault = 1.0
 )
 
+// LB variables for dual-stack
+var (
+	// Service.Spec.LoadBalancerIP has been deprecated and may be removed in a future release. Those two annotations are introduced as alternatives to set IPv4/IPv6 LoadBalancer IPs.
+	// Refer https://github.com/kubernetes/api/blob/3638040e4063e0f889c129220cd386497f328276/core/v1/types.go#L4459-L4468 for more details.
+	ServiceAnnotationLoadBalancerIPDualStack = map[bool]string{
+		false: "service.beta.kubernetes.io/azure-load-balancer-ipv4",
+		true:  "service.beta.kubernetes.io/azure-load-balancer-ipv6",
+	}
+)
+
 // load balancer
 const (
 	// PreConfiguredBackendPoolLoadBalancerTypesNone means that the load balancers are not pre-configured
diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go
index 203125904..adbdbd201 100644
--- a/pkg/provider/azure_loadbalancer.go
+++ b/pkg/provider/azure_loadbalancer.go
@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"net"
 	"reflect"
 	"sort"
 	"strconv"
@@ -45,6 +46,36 @@ import (
 	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
+// getServiceLoadBalancerIP retrieves LB IP from IPv4 annotation, then IPv6 annotation, then service.Spec.LoadBalancerIP.
+// TODO: Dual-stack support is not implemented.
+func getServiceLoadBalancerIP(service *v1.Service) string {
+	if service == nil {
+		return ""
+	}
+
+	if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]]; ok && ip != "" {
+		return ip
+	}
+	if ip, ok := service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]]; ok && ip != "" {
+		return ip
+	}
+
+	// Retrieve LB IP from service.Spec.LoadBalancerIP (will be deprecated)
+	return service.Spec.LoadBalancerIP
+}
+
+// setServiceLoadBalancerIP sets LB IP to a Service
+func setServiceLoadBalancerIP(service *v1.Service, ip string) {
+	if service.Annotations == nil {
+		service.Annotations = map[string]string{}
+	}
+	if net.ParseIP(ip).To4() != nil {
+		service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = ip
+		return
+	}
+	service.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = ip
+}
+
 // GetLoadBalancer returns whether the specified load balancer and its components exist, and
 // if so, what its status is.
 func (az *Cloud) GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) {
@@ -1052,7 +1083,7 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service,
 	}
 
 	pipResourceGroup := az.getPublicIPAddressResourceGroup(service)
-	loadBalancerIP := service.Spec.LoadBalancerIP
+	loadBalancerIP := getServiceLoadBalancerIP(service)
 
 	// Assume that the service without loadBalancerIP set is a primary service.
 	// If a secondary service doesn't set the loadBalancerIP, it is not allowed to share the IP.
@@ -1110,14 +1141,15 @@ func flipServiceInternalAnnotation(service *v1.Service) *v1.Service {
 func updateServiceLoadBalancerIP(service *v1.Service, serviceIP string) *v1.Service {
 	copyService := service.DeepCopy()
 	if len(serviceIP) > 0 && copyService != nil {
-		copyService.Spec.LoadBalancerIP = serviceIP
+		setServiceLoadBalancerIP(copyService, serviceIP)
 	}
 	return copyService
 }
 
 func (az *Cloud) findServiceIPAddress(ctx context.Context, clusterName string, service *v1.Service) (string, error) {
-	if len(service.Spec.LoadBalancerIP) > 0 {
-		return service.Spec.LoadBalancerIP, nil
+	lbIP := getServiceLoadBalancerIP(service)
+	if len(lbIP) > 0 {
+		return lbIP, nil
 	}
 
 	if len(service.Status.LoadBalancer.Ingress) > 0 && len(service.Status.LoadBalancer.Ingress[0].IP) > 0 {
@@ -1469,7 +1501,7 @@ func (az *Cloud) isFrontendIPChanged(clusterName string, config network.Frontend
 	if !strings.EqualFold(to.String(config.Name), lbFrontendIPConfigName) {
 		return false, nil
 	}
-	loadBalancerIP := service.Spec.LoadBalancerIP
+	loadBalancerIP := getServiceLoadBalancerIP(service)
 	isInternal := requiresInternalLoadBalancer(service)
 	if isInternal {
 		// Judge subnet
@@ -1987,7 +2019,7 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv
 					configProperties.PrivateIPAddressVersion = network.IPVersionIPv6
 				}
 
-				loadBalancerIP := service.Spec.LoadBalancerIP
+				loadBalancerIP := getServiceLoadBalancerIP(service)
 				if loadBalancerIP != "" {
 					configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic
 					configProperties.PrivateIPAddress = &loadBalancerIP
@@ -3541,7 +3573,7 @@ func getServiceTags(service *v1.Service) []string {
 // The pip is user-created if and only if there is no service tags.
 // The service owns the pip if:
 // 1. The serviceName is included in the service tags of a system-created pip.
-// 2. The service.Spec.LoadBalancerIP matches the IP address of a user-created pip.
+// 2. The service LoadBalancerIP matches the IP address of a user-created pip.
 func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clusterName string) (bool, bool) {
 	if service == nil || pip == nil {
 		klog.Warningf("serviceOwnsPublicIP: nil service or public IP")
@@ -3561,7 +3593,7 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus
 
 		// if there is no service tag on the pip, it is user-created pip
 		if to.String(serviceTag) == "" {
-			return strings.EqualFold(to.String(pip.IPAddress), service.Spec.LoadBalancerIP), true
+			return strings.EqualFold(to.String(pip.IPAddress), getServiceLoadBalancerIP(service)), true
 		}
 
 		if serviceTag != nil {
@@ -3580,7 +3612,7 @@ func serviceOwnsPublicIP(service *v1.Service, pip *network.PublicIPAddress, clus
 			} else {
 				// if the service is not included in te tags of the system-created pip, check the ip address
 				// this could happen for secondary services
-				return strings.EqualFold(to.String(pip.IPAddress), service.Spec.LoadBalancerIP), false
+				return strings.EqualFold(to.String(pip.IPAddress), getServiceLoadBalancerIP(service)), false
 			}
 		}
 	}
diff --git a/pkg/provider/azure_standard.go b/pkg/provider/azure_standard.go
index a0cb10722..c623287b7 100644
--- a/pkg/provider/azure_standard.go
+++ b/pkg/provider/azure_standard.go
@@ -353,7 +353,7 @@ func (az *Cloud) serviceOwnsFrontendIP(fip network.FrontendIPConfiguration, serv
 		return true, isPrimaryService, nil
 	}
 
-	loadBalancerIP := service.Spec.LoadBalancerIP
+	loadBalancerIP := getServiceLoadBalancerIP(service)
 	if loadBalancerIP == "" {
 		// it is a must that the secondary services set the loadBalancer IP
 		return false, isPrimaryService, nil
diff --git a/site/content/en/topics/pls-integration.md b/site/content/en/topics/pls-integration.md
new file mode 100644
index 000000000..7aabc16f3
--- /dev/null
+++ b/site/content/en/topics/pls-integration.md
@@ -0,0 +1,95 @@
+---
+title: "Azure Private Link Service Integration"
+linkTitle: "Azure Private Link Service Integration"
+type: docs
+description: >
+    Connect Azure Private Link service to Azure Standard Load Balancer.
+---
+
+Azure Private Link Service (PLS) is an infrastructure component that allows users to privately connect via a Private Endpoint (PE) in a VNET in Azure and a Frontend IP Configuration associated with an Azure Load Balancer (ALB). With Private Link, users as service providers can securely provide their services to consumers who can connect from within Azure or on-premises without data exfiltration risks.
+
+Before Private Link Service integration, users who wanted private connectivity from on-premises or other VNETs to their services in the Azure Kubernetes cluster were required to create a Private Link Service (PLS) to reference the Azure LoadBalancer. The user would then create a Private Endpoint (PE) to connect to the PLS to enable private connectivity. With this feature, a managed PLS to the LB would be created automatically, and the user would only be required to create PE connections to it for private connectivity.
+
+Currently, private link service connectivity is broken with Azure external Standard Load Balancer and floating ip enabled (default). To use managed private link service, users can either create an internal service by setting annotation `service.beta.kubernetes.io/azure-load-balancer-internal` to `true` or disable floating ip by setting annotation `service.beta.kubernetes.io/service.beta.kubernetes.io/azure-disable-load-balancer-floating-ip` to `true` ([more details here](../loadbalancer)).
+
+## PrivateLinkService annotations
+
+Below is a list of annotations supported for Kubernetes services with Azure PLS created:
+
+| Annotation | Value | Description | Required | Default |
+| ------------------------------------------------------------------------ | ---------------------------------- | ------------------------------------------------------------ |------|------|
+| `service.beta.kubernetes.io/azure-pls-create`                            | `"true"`                           | Boolean indicating whether a PLS needs to be created. | Required | |
+| `service.beta.kubernetes.io/azure-pls-name`                              | `<PLS name>`                       | String specifying the name of the PLS resource to be created. | Optional | `"pls-<LB frontend config name>"` |
+| `service.beta.kubernetes.io/azure-pls-ip-configuration-subnet`           |`<Subnet name>`                     | String indicating the subnet to which the PLS will be deployed. This subnet must exist in the same VNET as the backend pool. PLS NAT IPs are allocated within this subnet. | Optional | If `service.beta.kubernetes.io/azure-load-balancer-internal-subnet`, this ILB subnet is used. Otherwise, the default subnet from config file is used. |
+| `service.beta.kubernetes.io/azure-pls-ip-configuration-ip-address-count` | `[1-8]`                            | Total number of private NAT IPs to allocate. | Optional | 1 |
+| `service.beta.kubernetes.io/azure-pls-ip-configuration-ip-address`       | `"10.0.0.7 ... 10.0.0.10"`         | A space separated list of static **IPv4** IPs to be allocated. (IPv6 is not supported right now.) Total number of IPs should not be greater than the ip count specified in `service.beta.kubernetes.io/azure-pls-ip-configuration-ip-address-count`. If there are fewer IPs specified, the rest are dynamically allocated. The first IP in the list is set as `Primary`. |  Optional | All IPs are dynamically allocated. |
+| `service.beta.kubernetes.io/azure-pls-fqdns`                             | `"fqdn1 fqdn2"`                    | A space separated list of fqdns associated with the PLS. | Optional | `[]` |
+| `service.beta.kubernetes.io/azure-pls-proxy-protocol`                    | `"true"` or `"false"`              | Boolean indicating whether the TCP PROXY protocol should be enabled on the PLS to pass through connection information, including the link ID and source IP address. Note that the backend service MUST support the PROXY protocol or the connections will fail. | Optional | `false` |
+| `service.beta.kubernetes.io/azure-pls-visibility`                        | `"sub1 sub2 sub3 … subN"` or `"*"` | A space separated list of Azure subscription ids for which the private link service is visible. Use `"*"` to expose the PLS to all subs (Least restrictive). | Optional | Empty list `[]` indicating role-based access control only: This private link service will only be available to individuals with role-based access control permissions within your directory. (Most restrictive) |
+| `service.beta.kubernetes.io/azure-pls-auto-approval`                     | `"sub1 sub2 sub3 … subN"`          | A space separated list of Azure subscription ids. This allows PE connection requests from the subscriptions listed to the PLS to be automatically approved. This only works when visibility is set to "*". |  Optional | `[]` |
+
+For more details about each configuration, please refer to [Azure Private Link Service Documentation](https://docs.microsoft.com/en-us/cli/azure/network/private-link-service?view=azure-cli-latest#az-network-private-link-service-create).
+
+
+## Design Details
+
+### Creating managed PrivateLinkService
+
+When a `LoadBalancer` typed service is created without the annotations `service.beta.kubernetes.io/azure-load-balancer-ipv4`, `service.beta.kubernetes.io/azure-load-balancer-ipv6` or field `Service.Spec.LoadBalancerIP` set, an LB frontend IP configuration is created with a dynamically generated IP. If the service has the annotation `service.beta.kubernetes.io/azure-load-balancer-ipv4` or `service.beta.kubernetes.io/azure-load-balancer-ipv6` set, an existing LB frontend IP configuration may be reused if one exists; otherwise a static configuration is created with the specified IP. When a service is created with annotation `service.beta.kubernetes.io/azure-pls-create` set to `true` or updated later with the annotation added, a PLS resource attached to the LB frontend is created in the default resource group or the resource group user set in config file with key `PrivateLinkServiceResourceGroup`.
+
+The Kubernetes service creating the PLS is assigned as the owner of the resource. Azure cloud provider tags the PLS with cluster name and service name `kubernetes-owner-service: <namespace>/<service name>`. Only the owner service can later update the properties of the PLS resource.
+
+If there's a managed PLS already created for the LB frontend, the same PLS is reused automatically since each LB frontend can be referenced by only one PLS. If the LB frontend is attached to a user defined PLS, service creation should fail with proper error logged.
+
+For now, Azure cloud provider does not manage any [Private Link Endpoint](https://docs.microsoft.com/en-us/azure/private-link/private-endpoint-overview) resources. Once a PLS is created, users can create their own PEs to connect to the PLS.
+
+### Deleting managed PrivateLinkService
+
+Once a PLS is created, it shares the lifetime of the LB frontend IP configuration and is deleted only when its corresponding LB frontend gets deleted. As a result, a PLS may still exist even when its owner service is deleted. This is out of the consideration that multiple Kubernetes services can share the same LB frontend IP configuration and thus share the PLS automatically. More details are discussed in [next section](#sharing-managed-privatelinkservice).
+
+If there are active PE connections to the PLS, all connections are removed and the PEs become obsolete. Users are responsible for cleaning up the PE resources.
+
+### Sharing managed PrivateLinkService
+
+Multiple Kubernetes services can share the same LB frontend by specifying the same annotations `service.beta.kubernetes.io/azure-load-balancer-ipv4`, `service.beta.kubernetes.io/azure-load-balancer-ipv6` or field `Service.Spec.LoadBalancerIP` (for more details, please refer to [Multiple Services Sharing One IP Address](../shared-ip)). Once a PLS is attached to the LB frontend, these services automatically share the PLS. Users can access these services via the same PE but different ports.
+
+Azure cloud provider tags the service creating the PLS as the owner (`kubernetes-owner-service: <namespace>/<service name>`) and only allows that service to update the configurations of the PLS. If the owner service is deleted or if user wants some other service to take control, user can modify the tag value to a new service in `<namespace>/<service name>` pattern.
+
+PLS is only automatically deleted when the LB frontend IP configuration is deleted. One can delete a service while preserving the PLS by creating a temporary service referring to the same LB frontend.
+
+### Managed PrivateLinkService Creation example
+
+Below we provide an example for creating a Kubernetes service object with Azure ILB and PLS created:
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: myService
+  annotations:
+    service.beta.kubernetes.io/azure-load-balancer-internal: "true" # Use an internal LB with PLS
+    service.beta.kubernetes.io/azure-pls-create: "true"
+    service.beta.kubernetes.io/azure-pls-name: myServicePLS
+    service.beta.kubernetes.io/azure-pls-ip-configuration-subnet: pls-subnet
+    service.beta.kubernetes.io/azure-pls-ip-configuration-ip-address-count: "1"
+    service.beta.kubernetes.io/azure-pls-ip-configuration-ip-address: 10.240.0.9 # Must be available in pls-subnet
+    service.beta.kubernetes.io/azure-pls-fqdns: "fqdn1 fqdn2"
+    service.beta.kubernetes.io/azure-pls-proxy-protocol: "false"
+    service.beta.kubernetes.io/azure-pls-visibility: "*"
+    service.beta.kubernetes.io/azure-pls-auto-approval: "subId1"
+spec:
+  type: LoadBalancer
+  selector:
+    app: myApp
+  ports:
+    - name: myAppPort
+      protocol: TCP
+      port: 80
+      targetPort: 80
+```
+## Restrictions
+
+* PLS does not support basic Load Balancer or IP-based Load Balancer.
+* PLS connectivity is broken with Azure external Standard Load Balancer and floating ip enabled (default). Users can either use an internal Load Balancer or disable floating ip on loadbalancing rules (described above).
+* At this moment, PLS connectivity is broken if [AzureCNI V2](https://docs.microsoft.com/en-us/azure/aks/configure-azure-cni#dynamic-allocation-of-ips-and-enhanced-subnet-support) is used in cluster. Azure networking team is working on supporting the feature.
+* Due to limitation of [kubernetes#95555](https://github.com/kubernetes/kubernetes/issues/95555), when the service's externalTrafficPolicy set to Local, PLS need to use a different subnet from Pod's subnet. If the same subnet is required, then the service should use Cluster externalTrafficPolicy.
diff --git a/site/content/en/topics/shared-ip.md b/site/content/en/topics/shared-ip.md
index 2322d6640..ab5fb264c 100644
--- a/site/content/en/topics/shared-ip.md
+++ b/site/content/en/topics/shared-ip.md
@@ -27,7 +27,7 @@ spec:
   type: LoadBalancer
 ```
 
-Note that the `loadBalancerIP` is not set, or Azure would find a pre-allocated public IP with the address. After obtaining the IP address of the service, you could create other services using this address.
+Note that the annotations `service.beta.kubernetes.io/azure-load-balancer-ipv4`, `service.beta.kubernetes.io/azure-load-balancer-ipv6`, field `Service.Spec.LoadBalancerIP` are not set, or Azure would find a pre-allocated public IP with the address. After obtaining the IP address of the service, you could create other services using this address.
 
 ```yaml
 apiVersion: v1
@@ -35,8 +35,8 @@ kind: Service
 metadata:
   name: https
   namespace: default
+  service.beta.kubernetes.io/azure-load-balancer-ipv4: 1.2.3.4 # the IP address could be the same as it is of `nginx` service
 spec:
-  loadBalancerIP: 1.2.3.4 # the IP address could be the same as it is of `nginx` service
   ports:
     - port: 443
       protocol: TCP
@@ -46,7 +46,7 @@ spec:
   type: LoadBalancer
 ```
 
-Note that if you specify the `loadBalancerIP` but there is no corresponding public IP pre-allocated, an error would be reported.
+Note that if you specify the annotations `service.beta.kubernetes.io/azure-load-balancer-ipv4`, `service.beta.kubernetes.io/azure-load-balancer-ipv6` or field `Service.Spec.LoadBalancerIP` but there is no corresponding public IP pre-allocated, an error would be reported.
 
 ## DNS
 


Test patch:
diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go
index c38ce483a..a1e8f22b6 100644
--- a/pkg/provider/azure_loadbalancer_test.go
+++ b/pkg/provider/azure_loadbalancer_test.go
@@ -960,7 +960,7 @@ func TestServiceOwnsPublicIP(t *testing.T) {
 		t.Run(c.desc, func(t *testing.T) {
 			service := getTestService(c.serviceName, v1.ProtocolTCP, nil, false, 80)
 			if c.serviceLBIP != "" {
-				service.Spec.LoadBalancerIP = c.serviceLBIP
+				setServiceLoadBalancerIP(&service, c.serviceLBIP)
 			}
 			owns, isUserAssignedPIP := serviceOwnsPublicIP(&service, c.pip, c.clusterName)
 			assert.Equal(t, c.expectedOwns, owns, "TestCase[%d]: %s", i, c.desc)
@@ -2007,34 +2007,36 @@ func TestIsFrontendIPChanged(t *testing.T) {
 		},
 	}
 
-	for i, test := range testCases {
-		az := GetTestCloud(ctrl)
-		mockSubnetsClient := az.SubnetsClient.(*mocksubnetclient.MockInterface)
-		mockSubnetsClient.EXPECT().Get(gomock.Any(), "rg", "vnet", "testSubnet", "").Return(test.existingSubnet, nil).AnyTimes()
-		mockSubnetsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", "vnet", "testSubnet", test.existingSubnet).Return(nil)
-		err := az.SubnetsClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "testSubnet", test.existingSubnet)
-		if err != nil {
-			t.Fatalf("TestCase[%d] meets unexpected error: %v", i, err)
-		}
-
-		mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface)
-		mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
-		for _, existingPIP := range test.existingPIPs {
-			mockPIPsClient.EXPECT().Get(gomock.Any(), "rg", *existingPIP.Name, gomock.Any()).Return(existingPIP, nil).AnyTimes()
-			err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", *existingPIP.Name, existingPIP)
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			az := GetTestCloud(ctrl)
+			mockSubnetsClient := az.SubnetsClient.(*mocksubnetclient.MockInterface)
+			mockSubnetsClient.EXPECT().Get(gomock.Any(), "rg", "vnet", "testSubnet", "").Return(test.existingSubnet, nil).AnyTimes()
+			mockSubnetsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", "vnet", "testSubnet", test.existingSubnet).Return(nil)
+			err := az.SubnetsClient.CreateOrUpdate(context.TODO(), "rg", "vnet", "testSubnet", test.existingSubnet)
 			if err != nil {
-				t.Fatalf("TestCase[%d] meets unexpected error: %v", i, err)
+				t.Fatal(err)
 			}
-		}
-		test.service.Spec.LoadBalancerIP = test.loadBalancerIP
-		test.service.Annotations[consts.ServiceAnnotationLoadBalancerInternalSubnet] = test.annotations
-		flag, rerr := az.isFrontendIPChanged("testCluster", test.config,
-			&test.service, test.lbFrontendIPConfigName, &test.existingPIPs)
-		if rerr != nil {
-			fmt.Println(rerr.Error())
-		}
-		assert.Equal(t, test.expectedFlag, flag, "TestCase[%d]: %s", i, test.desc)
-		assert.Equal(t, test.expectedError, rerr != nil, "TestCase[%d]: %s", i, test.desc)
+
+			mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface)
+			mockPIPsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
+			for _, existingPIP := range test.existingPIPs {
+				mockPIPsClient.EXPECT().Get(gomock.Any(), "rg", *existingPIP.Name, gomock.Any()).Return(existingPIP, nil).AnyTimes()
+				err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", *existingPIP.Name, existingPIP)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			setServiceLoadBalancerIP(&test.service, test.loadBalancerIP)
+			test.service.Annotations[consts.ServiceAnnotationLoadBalancerInternalSubnet] = test.annotations
+			flag, rerr := az.isFrontendIPChanged("testCluster", test.config,
+				&test.service, test.lbFrontendIPConfigName, &test.existingPIPs)
+			if rerr != nil {
+				fmt.Println(rerr.Error())
+			}
+			assert.Equal(t, test.expectedFlag, flag)
+			assert.Equal(t, test.expectedError, rerr != nil)
+		})
 	}
 }
 
@@ -2081,7 +2083,7 @@ func TestDeterminePublicIPName(t *testing.T) {
 	for i, test := range testCases {
 		az := GetTestCloud(ctrl)
 		service := getTestService("test1", v1.ProtocolTCP, nil, false, 80)
-		service.Spec.LoadBalancerIP = test.loadBalancerIP
+		setServiceLoadBalancerIP(&service, test.loadBalancerIP)
 
 		mockPIPsClient := az.PublicIPAddressesClient.(*mockpublicipclient.MockInterface)
 		mockPIPsClient.EXPECT().List(gomock.Any(), "rg").Return(test.existingPIPs, nil).MaxTimes(1)
@@ -3054,7 +3056,7 @@ func TestReconcileLoadBalancer(t *testing.T) {
 		clusterResources, expectedInterfaces, expectedVirtualMachines := getClusterResources(az, 3, 3)
 		setMockEnv(az, ctrl, expectedInterfaces, expectedVirtualMachines, 1)
 
-		test.service.Spec.LoadBalancerIP = "1.2.3.4"
+		setServiceLoadBalancerIP(&test.service, "1.2.3.4")
 
 		err := az.PublicIPAddressesClient.CreateOrUpdate(context.TODO(), "rg", "pipName", network.PublicIPAddress{
 			Name: to.StringPtr("pipName"),
@@ -4632,7 +4634,7 @@ func TestUnbindServiceFromPIP(t *testing.T) {
 	}
 	serviceName := "ns2/svc2"
 	service := getTestService(serviceName, v1.ProtocolTCP, nil, false, 80)
-	service.Spec.LoadBalancerIP = "1.2.3.4"
+	setServiceLoadBalancerIP(&service, "1.2.3.4")
 	expectedTags := []map[string]*string{
 		nil,
 		{consts.ServiceTagKey: to.StringPtr("")},
diff --git a/pkg/provider/azure_standard_test.go b/pkg/provider/azure_standard_test.go
index 64c9db508..644f7cafa 100644
--- a/pkg/provider/azure_standard_test.go
+++ b/pkg/provider/azure_standard_test.go
@@ -1684,10 +1684,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) {
 			},
 			service: &v1.Service{
 				ObjectMeta: meta.ObjectMeta{
-					UID: types.UID("secondary"),
-				},
-				Spec: v1.ServiceSpec{
-					LoadBalancerIP: "1.2.3.4",
+					UID:         types.UID("secondary"),
+					Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "1.2.3.4"},
 				},
 			},
 		},
@@ -1712,10 +1710,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) {
 			},
 			service: &v1.Service{
 				ObjectMeta: meta.ObjectMeta{
-					UID: types.UID("secondary"),
-				},
-				Spec: v1.ServiceSpec{
-					LoadBalancerIP: "4.3.2.1",
+					UID:         types.UID("secondary"),
+					Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"},
 				},
 			},
 		},
@@ -1739,10 +1735,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) {
 			},
 			service: &v1.Service{
 				ObjectMeta: meta.ObjectMeta{
-					UID: types.UID("secondary"),
-				},
-				Spec: v1.ServiceSpec{
-					LoadBalancerIP: "4.3.2.1",
+					UID:         types.UID("secondary"),
+					Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"},
 				},
 			},
 		},
@@ -1766,10 +1760,8 @@ func TestServiceOwnsFrontendIP(t *testing.T) {
 			},
 			service: &v1.Service{
 				ObjectMeta: meta.ObjectMeta{
-					UID: types.UID("secondary"),
-				},
-				Spec: v1.ServiceSpec{
-					LoadBalancerIP: "4.3.2.1",
+					UID:         types.UID("secondary"),
+					Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1"},
 				},
 			},
 			isOwned: true,
@@ -1784,11 +1776,11 @@ func TestServiceOwnsFrontendIP(t *testing.T) {
 			},
 			service: &v1.Service{
 				ObjectMeta: meta.ObjectMeta{
-					UID:         types.UID("secondary"),
-					Annotations: map[string]string{consts.ServiceAnnotationLoadBalancerInternal: "true"},
-				},
-				Spec: v1.ServiceSpec{
-					LoadBalancerIP: "4.3.2.1",
+					UID: types.UID("secondary"),
+					Annotations: map[string]string{
+						consts.ServiceAnnotationLoadBalancerInternal:           "true",
+						consts.ServiceAnnotationLoadBalancerIPDualStack[false]: "4.3.2.1",
+					},
 				},
 			},
 			isOwned: true,
diff --git a/pkg/provider/azure_test.go b/pkg/provider/azure_test.go
index 5be7a3c1c..cafaf02cf 100644
--- a/pkg/provider/azure_test.go
+++ b/pkg/provider/azure_test.go
@@ -688,7 +688,7 @@ func TestReconcileSecurityGroupFromAnyDestinationAddressPrefixToLoadBalancerIP(t
 
 	az := GetTestCloud(ctrl)
 	svc1 := getTestService("serviceea", v1.ProtocolTCP, nil, false, 80)
-	svc1.Spec.LoadBalancerIP = "192.168.0.0"
+	setServiceLoadBalancerIP(&svc1, "192.168.0.0")
 
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
@@ -698,7 +698,7 @@ func TestReconcileSecurityGroupFromAnyDestinationAddressPrefixToLoadBalancerIP(t
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -711,7 +711,7 @@ func TestReconcileSecurityGroupDynamicLoadBalancerIP(t *testing.T) {
 
 	az := GetTestCloud(ctrl)
 	svc1 := getTestService("servicea", v1.ProtocolTCP, nil, false, 80)
-	svc1.Spec.LoadBalancerIP = ""
+	setServiceLoadBalancerIP(&svc1, "")
 
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
@@ -1592,7 +1592,7 @@ func getTestServiceWithAnnotation(identifier string, annotations map[string]stri
 
 func getResourceGroupTestService(identifier, resourceGroup, loadBalancerIP string, requestedPorts ...int32) v1.Service {
 	svc := getTestService(identifier, v1.ProtocolTCP, nil, false, requestedPorts...)
-	svc.Spec.LoadBalancerIP = loadBalancerIP
+	setServiceLoadBalancerIP(&svc, loadBalancerIP)
 	svc.Annotations[consts.ServiceAnnotationLoadBalancerResourceGroup] = resourceGroup
 	return svc
 }
@@ -1815,7 +1815,7 @@ func validatePublicIP(t *testing.T, publicIP *network.PublicIPAddress, service *
 		t.Errorf("Expected publicIP resource has matching tags[%s]", consts.ClusterNameKey)
 	}
 
-	// We cannot use service.Spec.LoadBalancerIP to compare with
+	// We cannot use Service LoadBalancerIP to compare with
 	// Public IP's IPAddress
 	// Because service properties are updated outside of cloudprovider code
 }
@@ -1879,6 +1879,7 @@ func validateSecurityGroup(t *testing.T, securityGroup *network.SecurityGroup, s
 	az := GetTestCloud(ctrl)
 	seenRules := make(map[string]string)
 	for i, svc := range services {
+		svc := svc
 		for _, wantedRule := range svc.Spec.Ports {
 			sources := getServiceSourceRanges(&services[i])
 			for _, source := range sources {
@@ -1887,7 +1888,7 @@ func validateSecurityGroup(t *testing.T, securityGroup *network.SecurityGroup, s
 				foundRule := false
 				for _, actualRule := range *securityGroup.SecurityRules {
 					if strings.EqualFold(*actualRule.Name, wantedRuleName) {
-						err := securityRuleMatches(source, wantedRule, svc.Spec.LoadBalancerIP, actualRule)
+						err := securityRuleMatches(source, wantedRule, getServiceLoadBalancerIP(&svc), actualRule)
 						if err != nil {
 							t.Errorf("Found matching security rule %q but properties were incorrect: %v", wantedRuleName, err)
 						}
@@ -2322,13 +2323,13 @@ func TestIfServiceSpecifiesSharedRuleAndRuleDoesNotExistItIsCreated(t *testing.T
 
 	az := GetTestCloud(ctrl)
 	svc := getTestService("servicea", v1.ProtocolTCP, nil, false, 80)
-	svc.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc, testIP1)
 	svc.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), nil, true)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(getServiceLoadBalancerIP(&svc)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -2365,7 +2366,7 @@ func TestIfServiceSpecifiesSharedRuleAndRuleExistsThenTheServicesPortAndAddressA
 
 	az := GetTestCloud(ctrl)
 	svc := getTestService("servicesr", v1.ProtocolTCP, nil, false, 80)
-	svc.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc, testIP1)
 	svc.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	expectedRuleName := testRuleName
@@ -2387,7 +2388,7 @@ func TestIfServiceSpecifiesSharedRuleAndRuleExistsThenTheServicesPortAndAddressA
 	}
 	setMockSecurityGroup(az, ctrl, sg)
 
-	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(svc.Spec.LoadBalancerIP), nil, true)
+	sg, err := az.reconcileSecurityGroup(testClusterName, &svc, to.StringPtr(getServiceLoadBalancerIP(&svc)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error: %q", err)
 	}
@@ -2421,22 +2422,22 @@ func TestIfServicesSpecifySharedRuleButDifferentPortsThenSeparateRulesAreCreated
 	az := GetTestCloud(ctrl)
 
 	svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444)
-	svc1.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc1, testIP1)
 	svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888)
-	svc2.Spec.LoadBalancerIP = testIP2
+	setServiceLoadBalancerIP(&svc2, testIP2)
 	svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
@@ -2490,11 +2491,12 @@ func TestIfServicesSpecifySharedRuleButDifferentProtocolsThenSeparateRulesAreCre
 	az := GetTestCloud(ctrl)
 
 	svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444)
-	svc1.Spec.LoadBalancerIP = testIP1
+
+	setServiceLoadBalancerIP(&svc1, testIP1)
 	svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc2 := getTestService("servicesr2", v1.ProtocolUDP, nil, false, 4444)
-	svc2.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc2, testIP1)
 	svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	testRuleName3 := "shared-UDP-4444-Internet"
@@ -2502,12 +2504,12 @@ func TestIfServicesSpecifySharedRuleButDifferentProtocolsThenSeparateRulesAreCre
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
@@ -2559,12 +2561,12 @@ func TestIfServicesSpecifySharedRuleButDifferentSourceAddressesThenSeparateRules
 	az := GetTestCloud(ctrl)
 
 	svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 80)
-	svc1.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc1, testIP1)
 	svc1.Spec.LoadBalancerSourceRanges = []string{"192.168.12.0/24"}
 	svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 80)
-	svc2.Spec.LoadBalancerIP = testIP2
+	setServiceLoadBalancerIP(&svc2, testIP2)
 	svc2.Spec.LoadBalancerSourceRanges = []string{"192.168.34.0/24"}
 	svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
@@ -2574,12 +2576,12 @@ func TestIfServicesSpecifySharedRuleButDifferentSourceAddressesThenSeparateRules
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
@@ -2633,32 +2635,32 @@ func TestIfServicesSpecifySharedRuleButSomeAreOnDifferentPortsThenRulesAreSepara
 	az := GetTestCloud(ctrl)
 
 	svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444)
-	svc1.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc1, testIP1)
 	svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888)
-	svc2.Spec.LoadBalancerIP = testIP2
+	setServiceLoadBalancerIP(&svc2, testIP2)
 	svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444)
-	svc3.Spec.LoadBalancerIP = testIP3
+	setServiceLoadBalancerIP(&svc3, testIP3)
 	svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	testRuleName23 := testRuleName2
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc3: %q", err)
 	}
@@ -2734,11 +2736,11 @@ func TestIfServiceSpecifiesSharedRuleAndServiceIsDeletedThenTheServicesPortAndAd
 	az := GetTestCloud(ctrl)
 
 	svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 80)
-	svc1.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc1, testIP1)
 	svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 80)
-	svc2.Spec.LoadBalancerIP = testIP2
+	setServiceLoadBalancerIP(&svc2, testIP2)
 	svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	expectedRuleName := testRuleName
@@ -2746,19 +2748,19 @@ func TestIfServiceSpecifiesSharedRuleAndServiceIsDeletedThenTheServicesPortAndAd
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
 
 	validateSecurityGroup(t, sg, svc1, svc2)
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc1: %q", err)
 	}
@@ -2792,39 +2794,39 @@ func TestIfSomeServicesShareARuleAndOneIsDeletedItIsRemovedFromTheRightRule(t *t
 	az := GetTestCloud(ctrl)
 
 	svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444)
-	svc1.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc1, testIP1)
 	svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888)
-	svc2.Spec.LoadBalancerIP = testIP2
+	setServiceLoadBalancerIP(&svc2, testIP2)
 	svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444)
-	svc3.Spec.LoadBalancerIP = testIP3
+	setServiceLoadBalancerIP(&svc3, testIP3)
 	svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	testRuleName23 := testRuleName2
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc3: %q", err)
 	}
 
 	validateSecurityGroup(t, sg, svc1, svc2, svc3)
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc1: %q", err)
 	}
@@ -2900,44 +2902,44 @@ func TestIfServiceSpecifiesSharedRuleAndLastServiceIsDeletedThenRuleIsDeleted(t
 	az := GetTestCloud(ctrl)
 
 	svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444)
-	svc1.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc1, testIP1)
 	svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888)
-	svc2.Spec.LoadBalancerIP = testIP2
+	setServiceLoadBalancerIP(&svc2, testIP2)
 	svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444)
-	svc3.Spec.LoadBalancerIP = testIP3
+	setServiceLoadBalancerIP(&svc3, testIP3)
 	svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	testRuleName23 := testRuleName2
 	sg := getTestSecurityGroup(az)
 	setMockSecurityGroup(az, ctrl, sg)
 
-	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, true)
+	_, err := az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc1: %q", err)
 	}
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(svc2.Spec.LoadBalancerIP), nil, true)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc2, to.StringPtr(getServiceLoadBalancerIP(&svc2)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc2: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, true)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, true)
 	if err != nil {
 		t.Errorf("Unexpected error adding svc3: %q", err)
 	}
 
 	validateSecurityGroup(t, sg, svc1, svc2, svc3)
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(svc3.Spec.LoadBalancerIP), nil, false)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc3, to.StringPtr(getServiceLoadBalancerIP(&svc3)), nil, false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc3: %q", err)
 	}
@@ -2982,23 +2984,23 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) {
 	var err error
 
 	svc1 := getTestService("servicesr1", v1.ProtocolTCP, nil, false, 4444)
-	svc1.Spec.LoadBalancerIP = testIP1
+	setServiceLoadBalancerIP(&svc1, testIP1)
 	svc1.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc2 := getTestService("servicesr2", v1.ProtocolTCP, nil, false, 8888)
-	svc2.Spec.LoadBalancerIP = testIP2
+	setServiceLoadBalancerIP(&svc2, testIP2)
 	svc2.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc3 := getTestService("servicesr3", v1.ProtocolTCP, nil, false, 4444)
-	svc3.Spec.LoadBalancerIP = testIP3
+	setServiceLoadBalancerIP(&svc3, testIP3)
 	svc3.Annotations[consts.ServiceAnnotationSharedSecurityRule] = consts.TrueAnnotationValue
 
 	svc4 := getTestService("servicesr4", v1.ProtocolTCP, nil, false, 4444)
-	svc4.Spec.LoadBalancerIP = "192.168.22.33"
+	setServiceLoadBalancerIP(&svc4, "192.168.22.33")
 	svc4.Annotations[consts.ServiceAnnotationSharedSecurityRule] = "false"
 
 	svc5 := getTestService("servicesr5", v1.ProtocolTCP, nil, false, 8888)
-	svc5.Spec.LoadBalancerIP = "192.168.22.33"
+	setServiceLoadBalancerIP(&svc5, "192.168.22.33")
 	svc5.Annotations[consts.ServiceAnnotationSharedSecurityRule] = "false"
 
 	testServices := []v1.Service{svc1, svc2, svc3, svc4, svc5}
@@ -3011,7 +3013,8 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) {
 	setMockSecurityGroup(az, ctrl, sg)
 
 	for i, svc := range testServices {
-		_, err := az.reconcileSecurityGroup(testClusterName, &testServices[i], to.StringPtr(svc.Spec.LoadBalancerIP), nil, true)
+		svc := svc
+		_, err := az.reconcileSecurityGroup(testClusterName, &testServices[i], to.StringPtr(getServiceLoadBalancerIP(&svc)), nil, true)
 		if err != nil {
 			t.Errorf("Unexpected error adding svc%d: %q", i+1, err)
 		}
@@ -3086,8 +3089,8 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) {
 	if securityRule4.DestinationAddressPrefix == nil {
 		t.Errorf("Expected unshared rule %s to have a destination IP address", expectedRuleName4)
 	} else {
-		if !strings.EqualFold(*securityRule4.DestinationAddressPrefix, svc4.Spec.LoadBalancerIP) {
-			t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName4, svc4.Spec.LoadBalancerIP, *securityRule4.DestinationAddressPrefix)
+		if !strings.EqualFold(*securityRule4.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc4)) {
+			t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName4, getServiceLoadBalancerIP(&svc4), *securityRule4.DestinationAddressPrefix)
 		}
 	}
 
@@ -3098,17 +3101,17 @@ func TestCanCombineSharedAndPrivateRulesInSameGroup(t *testing.T) {
 	if securityRule5.DestinationAddressPrefix == nil {
 		t.Errorf("Expected unshared rule %s to have a destination IP address", expectedRuleName5)
 	} else {
-		if !strings.EqualFold(*securityRule5.DestinationAddressPrefix, svc5.Spec.LoadBalancerIP) {
-			t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName5, svc5.Spec.LoadBalancerIP, *securityRule5.DestinationAddressPrefix)
+		if !strings.EqualFold(*securityRule5.DestinationAddressPrefix, getServiceLoadBalancerIP(&svc5)) {
+			t.Errorf("Expected unshared rule %s to have a destination %s but had %s", expectedRuleName5, getServiceLoadBalancerIP(&svc5), *securityRule5.DestinationAddressPrefix)
 		}
 	}
 
-	_, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(svc1.Spec.LoadBalancerIP), nil, false)
+	_, err = az.reconcileSecurityGroup(testClusterName, &svc1, to.StringPtr(getServiceLoadBalancerIP(&svc1)), nil, false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc1: %q", err)
 	}
 
-	sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, to.StringPtr(svc5.Spec.LoadBalancerIP), nil, false)
+	sg, err = az.reconcileSecurityGroup(testClusterName, &svc5, to.StringPtr(getServiceLoadBalancerIP(&svc5)), nil, false)
 	if err != nil {
 		t.Errorf("Unexpected error removing svc5: %q", err)
 	}
diff --git a/tests/e2e/network/ensureloadbalancer.go b/tests/e2e/network/ensureloadbalancer.go
index df90f765e..b3d41b4e1 100644
--- a/tests/e2e/network/ensureloadbalancer.go
+++ b/tests/e2e/network/ensureloadbalancer.go
@@ -19,6 +19,7 @@ package network
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"strconv"
 	"strings"
@@ -158,7 +159,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 	It("should support BYO public IP", func() {
 		By("creating a public IP with tags")
 		ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4]
-		pip := defaultPublicIPAddress(ipName)
+		pip := defaultPublicIPAddress(ipName, false)
 		expectedTags := map[string]*string{
 			"foo": to.StringPtr("bar"),
 		}
@@ -171,7 +172,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 
 		By("creating a service referencing the public IP")
 		service := utils.CreateLoadBalancerServiceManifest(testServiceName, nil, labels, ns.Name, ports)
-		service = updateServiceBalanceIP(service, false, targetIP)
+		service = updateServiceLBIP(service, false, targetIP)
 		_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "")
@@ -203,7 +204,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 		Expect(err).NotTo(HaveOccurred())
 		utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name)
 
-		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName))
+		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false))
 		Expect(err).NotTo(HaveOccurred())
 		targetIP := to.String(pip.IPAddress)
 		utils.Logf("PIP to %s", targetIP)
@@ -225,7 +226,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 		By("Updating service to bound to specific public IP")
 		utils.Logf("will update IP to %s", targetIP)
 		service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{})
-		service = updateServiceBalanceIP(service, false, targetIP)
+		service = updateServiceLBIP(service, false, targetIP)
 
 		_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
 		Expect(err).NotTo(HaveOccurred())
@@ -241,7 +242,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalTrue, labels, ns.Name, ports)
-		service = updateServiceBalanceIP(service, true, ip1)
+		service = updateServiceLBIP(service, true, ip1)
 		_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name)
@@ -269,7 +270,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 		By("Updating internal service private IP")
 		utils.Logf("will update IP to %s", ip2)
 		service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{})
-		service = updateServiceBalanceIP(service, true, ip2)
+		service = updateServiceLBIP(service, true, ip2)
 		_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 
@@ -287,7 +288,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 		Expect(err).NotTo(HaveOccurred())
 		utils.Logf("Successfully created LoadBalancer service " + testServiceName + " in namespace " + ns.Name)
 
-		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName))
+		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false))
 		Expect(err).NotTo(HaveOccurred())
 		targetIP := to.String(pip.IPAddress)
 
@@ -313,7 +314,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 		By("Updating service to bound to specific public IP")
 		utils.Logf("will update IP to %s, %v", targetIP, len(targetIP))
 		service, err = cs.CoreV1().Services(ns.Name).Get(context.TODO(), testServiceName, metav1.GetOptions{})
-		service = updateServiceBalanceIP(service, false, targetIP)
+		service = updateServiceLBIP(service, false, targetIP)
 
 		_, err = cs.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{})
 		Expect(err).NotTo(HaveOccurred())
@@ -326,12 +327,12 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 	It("should have no operation since no change in service when update", Label(utils.TestSuiteLabelSlow), func() {
 		suffix := string(uuid.NewUUID())[0:4]
 		ipName := basename + "-public-remain" + suffix
-		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName))
+		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false))
 		Expect(err).NotTo(HaveOccurred())
 		targetIP := to.String(pip.IPAddress)
 
 		service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationLoadBalancerInternalFalse, labels, ns.Name, ports)
-		service = updateServiceBalanceIP(service, false, targetIP)
+		service = updateServiceLBIP(service, false, targetIP)
 		_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		utils.Logf("Successfully created LoadBalancer service %s in namespace %s", testServiceName, ns.Name)
@@ -384,7 +385,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 
 	It("should support multiple external services sharing one preset public IP address", func() {
 		ipName := fmt.Sprintf("%s-public-remain-%s", basename, string(uuid.NewUUID())[0:4])
-		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName))
+		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false))
 		defer func() {
 			err = utils.DeletePIPWithRetry(tc, ipName, "")
 			Expect(err).NotTo(HaveOccurred())
@@ -420,7 +421,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 				TargetPort: intstr.FromInt(int(tcpPort)),
 			}}
 			service := utils.CreateLoadBalancerServiceManifest(serviceName, nil, serviceLabels, ns.Name, servicePort)
-			service = updateServiceBalanceIP(service, false, targetIP)
+			service = updateServiceLBIP(service, false, targetIP)
 			_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 			defer func() {
 				err = utils.DeleteService(cs, ns.Name, serviceName)
@@ -468,7 +469,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 			}}
 			service := utils.CreateLoadBalancerServiceManifest(serviceName, nil, serviceLabels, ns.Name, servicePort)
 			if sharedIP != "" {
-				service.Spec.LoadBalancerIP = sharedIP
+				service = updateServiceLBIP(service, false, sharedIP)
 			}
 			_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 			Expect(err).NotTo(HaveOccurred())
@@ -555,7 +556,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 			}}
 			service := utils.CreateLoadBalancerServiceManifest(serviceName, serviceAnnotationLoadBalancerInternalTrue, serviceLabels, ns.Name, servicePort)
 			if sharedIP != "" {
-				service.Spec.LoadBalancerIP = sharedIP
+				service = updateServiceLBIP(service, true, sharedIP)
 			}
 			_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 			defer func() {
@@ -619,7 +620,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 	It("should support disabling floating IP in load balancer rule with kubernetes service annotations", func() {
 		By("creating a public IP with tags")
 		ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4]
-		pip := defaultPublicIPAddress(ipName)
+		pip := defaultPublicIPAddress(ipName, false)
 		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip)
 		Expect(err).NotTo(HaveOccurred())
 		targetIP := to.String(pip.IPAddress)
@@ -627,7 +628,7 @@ var _ = Describe("Ensure LoadBalancer", Label(utils.TestSuiteLabelLB), func() {
 
 		By("creating a service referencing the public IP")
 		service := utils.CreateLoadBalancerServiceManifest(testServiceName, serviceAnnotationDisableLoadBalancerFloatingIP, labels, ns.Name, ports)
-		service = updateServiceBalanceIP(service, false, targetIP)
+		service = updateServiceLBIP(service, false, targetIP)
 		_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, testServiceName, "")
@@ -743,7 +744,7 @@ var _ = Describe("EnsureLoadBalancer should not update any resources when servic
 	It("should respect service with BYO public IP with various configurations", func() {
 		By("Creating a BYO public IP")
 		ipName := basename + "-public-IP" + string(uuid.NewUUID())[0:4]
-		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName))
+		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), defaultPublicIPAddress(ipName, false))
 		defer func() {
 			err = utils.DeletePIPWithRetry(tc, ipName, "")
 			Expect(err).NotTo(HaveOccurred())
@@ -992,12 +993,20 @@ func getLBBackendPoolIndex(lb *aznetwork.LoadBalancer) int {
 	return 0
 }
 
-func updateServiceBalanceIP(service *v1.Service, isInternal bool, ip string) (result *v1.Service) {
+func updateServiceLBIP(service *v1.Service, isInternal bool, ip string) (result *v1.Service) {
 	result = service
 	if result == nil {
 		return
 	}
-	result.Spec.LoadBalancerIP = ip
+	if result.Annotations == nil {
+		result.Annotations = map[string]string{}
+	}
+	if net.ParseIP(ip).To4() != nil {
+		result.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = ip
+	} else {
+		result.Annotations[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = ip
+	}
+
 	if judgeInternal(*service) == isInternal {
 		return
 	}
@@ -1009,7 +1018,7 @@ func updateServiceBalanceIP(service *v1.Service, isInternal bool, ip string) (re
 	return
 }
 
-func defaultPublicIPAddress(ipName string) aznetwork.PublicIPAddress {
+func defaultPublicIPAddress(ipName string, isIPv6 bool) aznetwork.PublicIPAddress {
 	// The default sku for LoadBalancer and PublicIP is basic.
 	skuName := aznetwork.PublicIPAddressSkuNameBasic
 	if skuEnv := os.Getenv(utils.LoadBalancerSkuEnv); skuEnv != "" {
@@ -1017,7 +1026,7 @@ func defaultPublicIPAddress(ipName string) aznetwork.PublicIPAddress {
 			skuName = aznetwork.PublicIPAddressSkuNameStandard
 		}
 	}
-	return aznetwork.PublicIPAddress{
+	pip := aznetwork.PublicIPAddress{
 		Name:     to.StringPtr(ipName),
 		Location: to.StringPtr(os.Getenv(utils.ClusterLocationEnv)),
 		Sku: &aznetwork.PublicIPAddressSku{
@@ -1027,6 +1036,10 @@ func defaultPublicIPAddress(ipName string) aznetwork.PublicIPAddress {
 			PublicIPAllocationMethod: aznetwork.IPAllocationMethodStatic,
 		},
 	}
+	if isIPv6 {
+		pip.PublicIPAddressPropertiesFormat.PublicIPAddressVersion = network.IPVersionIPv6
+	}
+	return pip
 }
 
 func defaultPublicIPPrefix(name string) aznetwork.PublicIPPrefix {
diff --git a/tests/e2e/network/network_security_group.go b/tests/e2e/network/network_security_group.go
index 4542e0f71..71f0721b6 100644
--- a/tests/e2e/network/network_security_group.go
+++ b/tests/e2e/network/network_security_group.go
@@ -289,7 +289,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func(
 	It("should support service annotation `service.beta.kubernetes.io/azure-disable-load-balancer-floating-ip`", func() {
 		By("Creating a public IP with tags")
 		ipName := basename + "-public-IP-disable-floating-ip"
-		pip := defaultPublicIPAddress(ipName)
+		pip := defaultPublicIPAddress(ipName, false)
 		pip, err := utils.WaitCreatePIP(tc, ipName, tc.GetResourceGroup(), pip)
 		Expect(err).NotTo(HaveOccurred())
 		targetIP := to.String(pip.IPAddress)
@@ -300,7 +300,7 @@ var _ = Describe("Network security group", Label(utils.TestSuiteLabelNSG), func(
 			consts.ServiceAnnotationDisableLoadBalancerFloatingIP: "true",
 		}
 		service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports)
-		service = updateServiceBalanceIP(service, false, targetIP)
+		service = updateServiceLBIP(service, false, targetIP)
 		_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		ip, err := utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, serviceName, "")
diff --git a/tests/e2e/network/private_link_service.go b/tests/e2e/network/private_link_service.go
index ca19b863a..32e5f7521 100644
--- a/tests/e2e/network/private_link_service.go
+++ b/tests/e2e/network/private_link_service.go
@@ -393,7 +393,7 @@ var _ = Describe("Private link service", Label(utils.TestSuiteLabelPrivateLinkSe
 			err = utils.DeleteService(cs, ns.Name, svc2)
 			Expect(err).NotTo(HaveOccurred())
 		}()
-		service2.Spec.LoadBalancerIP = ip
+		service2 = updateServiceLBIP(service2, true, ip)
 		_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service2, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		_, err = utils.WaitServiceExposureAndValidateConnectivity(cs, ns.Name, svc2, ip)
diff --git a/tests/e2e/network/service_annotations.go b/tests/e2e/network/service_annotations.go
index 63bc7dfc2..cc577f198 100644
--- a/tests/e2e/network/service_annotations.go
+++ b/tests/e2e/network/service_annotations.go
@@ -50,6 +50,8 @@ var (
 	scalesetRE               = regexp.MustCompile(`.*/subscriptions/(?:.*)/resourceGroups/(.+)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
 	lbNameRE                 = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Network/loadBalancers/(.+)/frontendIPConfigurations(?:.*)`)
 	backendIPConfigurationRE = regexp.MustCompile(`^/subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/virtualMachineScaleSets/(.+)/virtualMachines(?:.*)`)
+
+	ipFamily = utils.IPv4
 )
 
 const (
@@ -85,10 +87,6 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
 		ns, err = utils.CreateTestingNamespace(basename, cs)
 		Expect(err).NotTo(HaveOccurred())
 
-		utils.Logf("Creating Azure clients")
-		tc, err = utils.CreateAzureTestClient()
-		Expect(err).NotTo(HaveOccurred())
-
 		utils.Logf("Creating deployment " + serviceName)
 		deployment := createServerDeploymentManifest(serviceName, labels)
 		_, err = cs.AppsV1().Deployments(ns.Name).Create(context.TODO(), deployment, metav1.CreateOptions{})
@@ -101,6 +99,9 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
 		utils.Logf("Creating Azure clients")
 		tc, err = utils.CreateAzureTestClient()
 		Expect(err).NotTo(HaveOccurred())
+
+		ipFamily, err = utils.GetClusterServiceIPFamily()
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	AfterEach(func() {
@@ -250,7 +251,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
 
 		By("creating test PIP in the test resource group")
 		testPIPName := "testPIP-" + string(uuid.NewUUID())[0:4]
-		pip, err := utils.WaitCreatePIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName))
+		pip, err := utils.WaitCreatePIP(tc, testPIPName, *rg.Name, defaultPublicIPAddress(testPIPName, false))
 		Expect(err).NotTo(HaveOccurred())
 		defer func() {
 			utils.Logf("Cleaning up service and public IP")
@@ -265,7 +266,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
 		}
 		By("Creating service " + serviceName + " in namespace " + ns.Name)
 		service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports)
-		service.Spec.LoadBalancerIP = *pip.IPAddress
+		service = updateServiceLBIP(service, false, *pip.IPAddress)
 		_, err = cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		utils.Logf("Successfully created LoadBalancer service " + serviceName + " in namespace " + ns.Name)
@@ -285,6 +286,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
 			consts.ServiceAnnotationAzurePIPTags: "a=b,c= d,e =, =f",
 		}
 		service := utils.CreateLoadBalancerServiceManifest(serviceName, annotation, labels, ns.Name, ports)
+		service.GetAnnotations()
 		_, err := cs.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{})
 		Expect(err).NotTo(HaveOccurred())
 
@@ -337,7 +339,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
 	It("should support service annotation `service.beta.kubernetes.io/azure-pip-name`", func() {
 		By("Creating two test pips")
 		pipName1 := "pip1"
-		pip1, err := utils.WaitCreatePIP(tc, pipName1, tc.GetResourceGroup(), defaultPublicIPAddress(pipName1))
+		pip1, err := utils.WaitCreatePIP(tc, pipName1, tc.GetResourceGroup(), defaultPublicIPAddress(pipName1, false))
 		defer func() {
 			By("Cleaning up test PIP")
 			err := utils.DeletePIPWithRetry(tc, pipName1, tc.GetResourceGroup())
@@ -345,7 +347,7 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
 		}()
 		Expect(err).NotTo(HaveOccurred())
 		pipName2 := "pip2"
-		pip2, err := utils.WaitCreatePIP(tc, pipName2, tc.GetResourceGroup(), defaultPublicIPAddress(pipName2))
+		pip2, err := utils.WaitCreatePIP(tc, pipName2, tc.GetResourceGroup(), defaultPublicIPAddress(pipName2, false))
 		defer func() {
 			By("Cleaning up test PIP")
 			err := utils.DeletePIPWithRetry(tc, pipName2, tc.GetResourceGroup())
@@ -553,6 +555,45 @@ var _ = Describe("Service with annotation", Label(utils.TestSuiteLabelServiceAnn
 		Expect((len(targetProbes))).To(Equal(1))
 		Expect(targetProbes[0].Protocol).To(Equal(network.ProbeProtocolHTTP))
 	})
+
+	// Check if the following annotations are correctly set with Service LB IP
+	// service.beta.kubernetes.io/azure-load-balancer-ipv4 or service.beta.kubernetes.io/azure-load-balancer-ipv6
+	It("should support service annotation 'service.beta.kubernetes.io/azure-load-balancer-ip'", func() {
+		pipName := fmt.Sprintf("%s-public-IP%s", basename, string(uuid.NewUUID())[0:4])
+		By(fmt.Sprintf("Creating a public IP %q", pipName))
+		var pip network.PublicIPAddress
+		if ipFamily == utils.IPv4 {
+			pip = defaultPublicIPAddress(pipName, false)
+		} else if ipFamily == utils.IPv6 {
+			pip = defaultPublicIPAddress(pipName, true)
+		} else {
+			// TODO: dual-stack support
+		}
+		pip, err := utils.WaitCreatePIP(tc, pipName, tc.GetResourceGroup(), pip)
+		Expect(err).NotTo(HaveOccurred())
+		pipAddr := to.String(pip.IPAddress)
+		utils.Logf("Created pip with address %s", pipAddr)
+
+		annotation := map[string]string{}
+		if ipFamily == utils.IPv4 {
+			annotation[consts.ServiceAnnotationLoadBalancerIPDualStack[false]] = pipAddr
+		} else if ipFamily == utils.IPv6 {
+			annotation[consts.ServiceAnnotationLoadBalancerIPDualStack[true]] = pipAddr
+		} else {
+			// TODO: dual-stack support
+		}
+
+		By("Creating a Service")
+		publicIP := createAndExposeDefaultServiceWithAnnotation(cs, serviceName, ns.Name, labels, annotation, ports)
+		defer func() {
+			By("Cleaning up service")
+			err := utils.DeleteService(cs, ns.Name, serviceName)
+			Expect(err).NotTo(HaveOccurred())
+		}()
+
+		By("Check if the Service has the correct address")
+		Expect(publicIP).To(Equal(pipAddr))
+	})
 })
 
 var _ = Describe("Multiple VMSS", Label(utils.TestSuiteLabelMultiNodePools, utils.TestSuiteLabelVMSS), func() {
diff --git a/tests/e2e/utils/kubectl.go b/tests/e2e/utils/kubectl.go
index 02042a105..1fd6efa73 100644
--- a/tests/e2e/utils/kubectl.go
+++ b/tests/e2e/utils/kubectl.go
@@ -100,7 +100,9 @@ func KubectlCmd(namespace string, args ...string) *exec.Cmd {
 	Logf("Kubernetes configuration file name: %s", filename)
 
 	defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+filename)
-	defaultArgs = append(defaultArgs, fmt.Sprintf("--namespace=%s", namespace))
+	if namespace != "" {
+		defaultArgs = append(defaultArgs, fmt.Sprintf("--namespace=%s", namespace))
+	}
 	kubectlArgs := append(defaultArgs, args...)
 
 	cmd := exec.Command("kubectl", kubectlArgs...)
diff --git a/tests/e2e/utils/network_utils.go b/tests/e2e/utils/network_utils.go
index aa2547d5c..f535ad3ef 100644
--- a/tests/e2e/utils/network_utils.go
+++ b/tests/e2e/utils/network_utils.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"regexp"
 	"strings"
 	"time"
 
@@ -32,6 +33,14 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
+type IPFamily string
+
+var (
+	IPv4      IPFamily = "ipv4"
+	IPv6      IPFamily = "ipv6"
+	DualStack IPFamily = "dualstack"
+)
+
 // getVirtualNetworkList returns the list of virtual networks in the cluster resource group.
 func (azureTestClient *AzureTestClient) getVirtualNetworkList() (result aznetwork.VirtualNetworkListResultPage, err error) {
 	Logf("Getting virtual network list")
@@ -476,3 +485,58 @@ func (azureTestClient *AzureTestClient) ListPrivateLinkServices(resourceGroupNam
 
 	return result, nil
 }
+
+func retrieveCIDRs(cmd string, reg string) ([]string, error) {
+	res := make([]string, 2)
+	stdout, err := RunKubectl("", strings.Split(cmd, " ")...)
+	if err != nil {
+		return res, fmt.Errorf("error when running the following kubectl command %q: %v, %s", cmd, err, stdout)
+	}
+	re := regexp.MustCompile(reg)
+	matches := re.FindStringSubmatch(stdout)
+	if len(matches) == 0 {
+		return res, fmt.Errorf("cannot retrieve CIDR, unexpected kubectl output: %s", stdout)
+	}
+	cidrs := strings.Split(matches[1], ",")
+	if len(cidrs) == 1 {
+		_, cidr, err := net.ParseCIDR(cidrs[0])
+		if err != nil {
+			return res, fmt.Errorf("CIDR cannot be parsed: %s", cidrs[0])
+		}
+		if cidr.IP.To4() != nil {
+			res[0] = cidrs[0]
+		} else {
+			res[1] = cidrs[0]
+		}
+	} else if len(cidrs) == 2 {
+		_, cidr, err := net.ParseCIDR(cidrs[0])
+		if err != nil {
+			return res, fmt.Errorf("CIDR cannot be parsed: %s", cidrs[0])
+		}
+		if cidr.IP.To4() != nil {
+			res[0] = cidrs[0]
+			res[1] = cidrs[1]
+		} else {
+			res[0] = cidrs[1]
+			res[1] = cidrs[0]
+		}
+	} else {
+		return res, fmt.Errorf("unexpected cluster CIDR: %s", matches[1])
+	}
+	return res, nil
+}
+
+// GetClusterServiceIPFamily gets cluster's Service IPFamily according to Service CIDRs.
+func GetClusterServiceIPFamily() (IPFamily, error) {
+	svcCIDRs, err := retrieveCIDRs("cluster-info dump | grep service-cluster-ip-range", `service-cluster-ip-range=([^"]+)`)
+	if err != nil {
+		return "", err
+	}
+	if svcCIDRs[0] != "" && svcCIDRs[1] == "" {
+		return IPv4, nil
+	}
+	if svcCIDRs[0] == "" && svcCIDRs[1] != "" {
+		return IPv6, nil
+	}
+	return DualStack, nil
+}