Repository: kubernetes-sigs/cloud-provider-azure (go)

Issue:
RulesUseSameBackendPortProtocolAndPoolAndFrontendIPWithFloatingIPEnabled error when using multiple ports with HA
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!
-->

**What happened**:

When using mutiple ports in service with annotation `service.beta.kubernetes.io/azure-load-balancer-enable-high-availability-ports: true`, the following error was reported:

```sh
Error syncing load balancer: failed to ensure load balancer: Retriable: false, RetryAfter: 0s, HTTPStatusCode: 400, RawError: Retriable: false, RetryAfter: 0s, HTTPStatusCode: 400, RawError: {
  "error": {
    "code": "RulesUseSameBackendPortProtocolAndPoolAndFrontendIPWithFloatingIPEnabled",
    "message": "Load balancing rules /subscriptions/<sub-id>/resourceGroups/<rg>/providers/Microsoft.Network/loadBalancers/kubernetes-internal/loadBalancingRules/a0de5d44226a749d393350b214d2c579-TCP-80 and /subscriptions/<sub-id>/resourceGroups/<rg>/providers/Microsoft.Network/loadBalancers/kubernetes-internal/loadBalancingRules/a0de5d44226a749d393350b214d2c579-TCP-443 with floating IP enabled use the same protocol All and backend port 0 and frontend IP /subscriptions/<sub-id>/resourceGroups/<rg>/providers/Microsoft.Network/loadBalancers/kubernetes-internal/frontendIPConfigurations/a0de5d44226a749d393350b214d2c579, and must not be used with the same backend address pool /subscriptions/<sub-id>/resourceGroups/<rg>/providers/Microsoft.Network/loadBalancers/kubernetes-internal/backendAddressPools/kubernetes.",
    "details": []
  }
}
```

**What you expected to happen**:

Errors shouldn't happen and the service should be created successfully.

**How to reproduce it**:

Create a service with multiple ports and enable HA ports feature.

**Anything else we need to know?**:

This is no such issue when only have one port.

**Environment**:
- Kubernetes version (use `kubectl version`): 1.20
- OS (e.g. from /etc/os-release):
- Kernel (e.g. `uname -a`):
- Install tools:
- Others:


Pull request description:
Ensure only LoadBalancer rule is created when HA mode is enabled
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/release.md#issue-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. Follow the instructions for writing a release note: https://git.k8s.io/community/contributors/guide/release-notes.md
6. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

**What type of PR is this?**
> Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line:
>
> /kind api-change
> /kind bug
> /kind cleanup
> /kind design
> /kind documentation
> /kind failing-test
> /kind feature
> /kind flake
/kind bug

**What this PR does / why we need it**:

When highAvailabilityPorts enabled, the LoadBalancer port is always using 0, hence only one rule should be configured.

**Which issue(s) this PR fixes**:
<!-- 
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #521 

**Special notes for your reviewer**:


**Release note**:
```
Ensure only LoadBalancer rule is created when HA mode is enabled
```


Patch:
diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go
index 094d2080a..8889372cb 100644
--- a/pkg/provider/azure_loadbalancer.go
+++ b/pkg/provider/azure_loadbalancer.go
@@ -1750,7 +1750,13 @@ func (az *Cloud) reconcileLoadBalancerRule(
 
 	var expectedProbes []network.Probe
 	var expectedRules []network.LoadBalancingRule
+	highAvailabilityPortsEnabled := false
 	for _, port := range ports {
+		if highAvailabilityPortsEnabled {
+			// Since the port is always 0 when enabling HA, only one rule should be configured.
+			break
+		}
+
 		lbRuleName := az.getLoadBalancerRuleName(service, port.Protocol, port.Port)
 		klog.V(2).Infof("reconcileLoadBalancerRule lb name (%s) rule name (%s)", lbName, lbRuleName)
 
@@ -1838,6 +1844,7 @@ func (az *Cloud) reconcileLoadBalancerRule(
 			expectedRule.FrontendPort = to.Int32Ptr(0)
 			expectedRule.BackendPort = to.Int32Ptr(0)
 			expectedRule.Protocol = network.TransportProtocolAll
+			highAvailabilityPortsEnabled = true
 		}
 
 		// we didn't construct the probe objects for UDP or SCTP because they're not allowed on Azure.


Test patch:
diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go
index 9ab439b8b..c5c975a88 100644
--- a/pkg/provider/azure_loadbalancer_test.go
+++ b/pkg/provider/azure_loadbalancer_test.go
@@ -1647,6 +1647,28 @@ func TestReconcileLoadBalancerRule(t *testing.T) {
 			expectedProbes:  getDefaultTestProbes("http", "/healthy"),
 			expectedRules:   getDefaultTestRules(true),
 		},
+		{
+			desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule (slb with HA enabled)",
+			service: getTestService("test1", v1.ProtocolTCP, map[string]string{
+				"service.beta.kubernetes.io/azure-load-balancer-enable-high-availability-ports": "true",
+				"service.beta.kubernetes.io/azure-load-balancer-internal":                       "true",
+			}, false, 80),
+			loadBalancerSku: "standard",
+			wantLb:          true,
+			expectedProbes:  getDefaultTestProbes("Tcp", ""),
+			expectedRules:   getHATestRules(true),
+		},
+		{
+			desc: "reconcileLoadBalancerRule shall return corresponding probe and lbRule (slb with HA enabled multi-ports services)",
+			service: getTestService("test1", v1.ProtocolTCP, map[string]string{
+				"service.beta.kubernetes.io/azure-load-balancer-enable-high-availability-ports": "true",
+				"service.beta.kubernetes.io/azure-load-balancer-internal":                       "true",
+			}, false, 80, 8080),
+			loadBalancerSku: "standard",
+			wantLb:          true,
+			expectedProbes:  getDefaultTestProbes("Tcp", ""),
+			expectedRules:   getHATestRules(true),
+		},
 	}
 	for i, test := range testCases {
 		az := GetTestCloud(ctrl)
@@ -1720,6 +1742,37 @@ func getDefaultTestRules(enableTCPReset bool) []network.LoadBalancingRule {
 	return expectedRules
 }
 
+func getHATestRules(enableTCPReset bool) []network.LoadBalancingRule {
+	expectedRules := []network.LoadBalancingRule{
+		{
+			Name: to.StringPtr("atest1-TCP-80"),
+			LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
+				Protocol: network.TransportProtocol("All"),
+				FrontendIPConfiguration: &network.SubResource{
+					ID: to.StringPtr("frontendIPConfigID"),
+				},
+				BackendAddressPool: &network.SubResource{
+					ID: to.StringPtr("backendPoolID"),
+				},
+				LoadDistribution:     "Default",
+				FrontendPort:         to.Int32Ptr(0),
+				BackendPort:          to.Int32Ptr(0),
+				EnableFloatingIP:     to.BoolPtr(true),
+				DisableOutboundSnat:  to.BoolPtr(false),
+				IdleTimeoutInMinutes: to.Int32Ptr(0),
+				Probe: &network.SubResource{
+					ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg/providers/" +
+						"Microsoft.Network/loadBalancers/lbname/probes/atest1-TCP-80"),
+				},
+			},
+		},
+	}
+	if enableTCPReset {
+		expectedRules[0].EnableTCPReset = to.BoolPtr(true)
+	}
+	return expectedRules
+}
+
 func getTestLoadBalancer(name, rgName, clusterName, identifier *string, service v1.Service, lbSku string) network.LoadBalancer {
 	lb := network.LoadBalancer{
 		Name: name,