Repository: kubernetes-sigs/cloud-provider-azure
Pull request #1289 (2022-03-18 13:00:59)

Fix bugs related to multiple slbs
<!--  Thanks for sending a pull request!  Here are some tips for you:

1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide/first-contribution.md#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
5. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?

<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

/kind bug

#### What this PR does / why we need it:

1. If the vmSet is changed from non-primary to the primary vmSet, we need to make sure the dedicated LB is deleted.
2. In some cases, the private IP of an internal service may change when switching from one LB to another. We need to re-use the previous private IP.

#### Which issue(s) this PR fixes:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

#### Special notes for your reviewer:

Bug 2 cannot be reproduced, I have just hit it once.

#### Does this PR introduce a user-facing change?
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->
```release-note
fix: remove agent pool LB if that vmSet is changed to primary vmSet
fix: reuse previous private IP address when changing load balancers
```

#### Additional documentation e.g., KEPs (Kubernetes Enhancement Proposals), usage docs, etc.:

<!--
This section can be blank if this pull request does not require a release note.

When adding links which point to resources within git repositories, like
KEPs or supporting documentation, please reference a specific commit and avoid
linking directly to the master branch. This ensures that links reference a
specific point in time, rather than a document that may change over time.

See here for guidance on getting permanent links to files: https://help.github.com/en/articles/getting-permanent-links-to-files

Please use the following format for linking documentation:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```


k8s-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubernetes-sigs/cloud-provider-azure/pull/1289#" title="Author self-approved">nilo19</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fcloud-provider-azure).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/OWNERS)~~ [nilo19]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
coveralls: 
[![Coverage Status](https://coveralls.io/builds/47491165/badge)](https://coveralls.io/builds/47491165)

Coverage increased (+0.03%) to 80.122% when pulling **89bb311d130d3cbd69751ecbfd913118aebf6c24 on nilo19:fix/primary-slb** into **66e1bec130df801212310a14bedae737fefd042e on kubernetes-sigs:master**.


Patch:
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index e92d4baaf..51694e2af 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -4,56 +4,46 @@ updates:
     directory: "/"
     schedule:
       interval: daily
-      time: "01:00"
-      timezone: "Asia/Shanghai"
     labels:
       - "area/dependency"
       - "release-note-none"
       - "ok-to-test"
-
+    open-pull-requests-limit: 5
+  
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
       interval: "daily"
-      time: "01:00"
-      timezone: "Asia/Shanghai"
-    labels:
-      - "area/dependency"
-      - "release-note-none"
-      - "ok-to-test"
   
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: daily
-      time: "01:00"
-      timezone: "Asia/Shanghai"
     labels:
       - "area/dependency"
       - "release-note-none"
       - "ok-to-test"
+    open-pull-requests-limit: 5
 
   - package-ecosystem: "npm"
     directory: "/site/"
     schedule:
       interval: daily
-      time: "01:00"
-      timezone: "Asia/Shanghai"
     labels:
       - "area/dependency"
       - "release-note-none"
       - "ok-to-test"
+    open-pull-requests-limit: 5
 
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
       interval: daily
-      time: "01:00"
-      timezone: "Asia/Shanghai"
     labels:
       - "area/dependency"
       - "release-note-none"
       - "ok-to-test"
+    open-pull-requests-limit: 5
     target-branch: "release-1.23"
     ignore:
       - dependency-name: "k8s.io/*"
@@ -63,12 +53,11 @@ updates:
     directory: "/"
     schedule:
       interval: daily
-      time: "01:00"
-      timezone: "Asia/Shanghai"
     labels:
       - "area/dependency"
       - "release-note-none"
       - "ok-to-test"
+    open-pull-requests-limit: 5
     target-branch: "release-1.1"
     ignore:
       - dependency-name: "k8s.io/*"
@@ -78,12 +67,11 @@ updates:
     directory: "/"
     schedule:
       interval: daily
-      time: "01:00"
-      timezone: "Asia/Shanghai"
     labels:
       - "area/dependency"
       - "release-note-none"
       - "ok-to-test"
+    open-pull-requests-limit: 5
     target-branch: "release-1.0"
     ignore:
       - dependency-name: "k8s.io/*"
@@ -93,12 +81,11 @@ updates:
     directory: "/"
     schedule:
       interval: daily
-      time: "01:00"
-      timezone: "Asia/Shanghai"
     labels:
       - "area/dependency"
       - "release-note-none"
       - "ok-to-test"
+    open-pull-requests-limit: 5
     target-branch: "release-0.7"
     ignore:
       - dependency-name: "k8s.io/*"
diff --git a/Dockerfile b/Dockerfile
index 14fe49d79..cffc47a65 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=linux/amd64 golang:1.18.0-buster AS builder
+FROM --platform=linux/amd64 golang:1.17-buster AS builder
 
 ARG ENABLE_GIT_COMMAND=true
 ARG ARCH=amd64
diff --git a/OWNERS b/OWNERS
index b07d4ffe0..ecb195e0e 100644
--- a/OWNERS
+++ b/OWNERS
@@ -3,14 +3,12 @@ approvers:
 - feiskyer
 - nilo19
 - MartinForReal
-- lzhecheng
 
 reviewers:
 - andyzhangx
 - feiskyer
 - nilo19
 - MartinForReal
-- lzhecheng
 
 emeritus_approvers:
 - brendandburns
diff --git a/README.md b/README.md
index a5d9695ba..47f81e2a6 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ azure-cloud-controller-manager \
     --cloud-provider=azure \
     --cluster-name=kubernetes \
     --controllers=*,-cloud-node \
-    --cloud-config=/etc/kubernetes/cloud-config/azure.json \
+    --cloud-config=/etc/kubernetes/azure.json \
     --kubeconfig=/etc/kubernetes/kubeconfig \
     --allocate-node-cidrs=true \
     --configure-cloud-routes=true \
@@ -77,7 +77,7 @@ azure-cloud-node-manager \
     --wait-routes=true
 ```
 
-It is recommended to run azure-cloud-controller-manager as Deployment with multiple replicas or Kubelet static Pods on each master Node. See [here](examples/out-of-tree/cloud-controller-manager.yaml) for the example.
+It is recommended to run azure-cloud-controller-manager as Pods on master nodes. See [here](examples/out-of-tree/cloud-controller-manager.yaml) for the example.
 
 Please checkout more details at [Deploy Cloud Controller Manager](http://kubernetes-sigs.github.io/cloud-provider-azure/install/azure-ccm/).
 
diff --git a/cloud-node-manager.Dockerfile b/cloud-node-manager.Dockerfile
index 73b4f41ed..217e328cb 100644
--- a/cloud-node-manager.Dockerfile
+++ b/cloud-node-manager.Dockerfile
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=linux/amd64 golang:1.18.0-buster AS builder
+FROM --platform=linux/amd64 golang:1.17-buster AS builder
 
 ARG ENABLE_GIT_COMMAND=true
 ARG ARCH=amd64
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index 0e8f1d220..0115e20ba 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -1,7 +1,7 @@
 # See https://cloud.google.com/cloud-build/docs/build-config
 
 # this must be specified in seconds. If omitted, defaults to 600s (10 mins)
-timeout: 7200s
+timeout: 2400s
 # this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
 # or any new substitutions added in the future.
 options:
diff --git a/examples/out-of-tree/cloud-controller-manager.yaml b/examples/out-of-tree/cloud-controller-manager.yaml
index b9d94e2b8..4ebc159f3 100644
--- a/examples/out-of-tree/cloud-controller-manager.yaml
+++ b/examples/out-of-tree/cloud-controller-manager.yaml
@@ -129,128 +129,68 @@ subjects:
     kind: User
     name: cloud-controller-manager
 ---
-apiVersion: apps/v1
-kind: Deployment
+apiVersion: v1
+kind: Pod
 metadata:
   name: cloud-controller-manager
   namespace: kube-system
   labels:
+    tier: control-plane
     component: cloud-controller-manager
 spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      tier: control-plane
-      component: cloud-controller-manager
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        component: cloud-controller-manager
-    spec:
-      priorityClassName: system-node-critical
-      hostNetwork: true
-      serviceAccountName: cloud-controller-manager
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-      containers:
-        - name: cloud-controller-manager
-          image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.23.5
-          imagePullPolicy: IfNotPresent
-          command: ["cloud-controller-manager"]
-          args:
-            - "--allocate-node-cidrs=true" # "false" for Azure CNI and "true" for other network plugins
-            - "--cloud-config=/etc/kubernetes/cloud-config/azure.json"
-            - "--cloud-provider=azure"
-            - "--cluster-cidr=10.244.0.0/16"
-            - "--cluster-name=k8s"
-            - "--controllers=*,-cloud-node" # disable cloud-node controller
-            - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins
-            - "--leader-elect=true"
-            - "--route-reconciliation-period=10s"
-            - "--v=4"
-            - "--port=10267"
-          resources:
-            requests:
-              cpu: 100m
-              memory: 128Mi
-            limits:
-              cpu: "4"
-              memory: 2Gi
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: 10267
-            initialDelaySeconds: 20
-            periodSeconds: 10
-            timeoutSeconds: 5
-          volumeMounts:
-            - name: etc-kubernetes
-              mountPath: /etc/kubernetes
-            - name: cloud-config
-              mountPath: /etc/kubernetes/cloud-config
-              readOnly: true
-            - name: etc-ssl
-              mountPath: /etc/ssl
-              readOnly: true
-            - name: msi
-              mountPath: /var/lib/waagent/ManagedIdentity-Settings
-              readOnly: true
-      volumes:
+  priorityClassName: system-node-critical
+  hostNetwork: true
+  serviceAccountName: cloud-controller-manager
+  tolerations:
+    - key: node-role.kubernetes.io/master
+      effect: NoSchedule
+  containers:
+    - name: cloud-controller-manager
+      image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.23.5
+      imagePullPolicy: IfNotPresent
+      command: ["cloud-controller-manager"]
+      args:
+        - "--allocate-node-cidrs=true" # "false" for Azure CNI and "true" for other network plugins
+        - "--cloud-config=/etc/kubernetes/azure.json"
+        - "--cloud-provider=azure"
+        - "--cluster-cidr=10.244.0.0/16"
+        - "--cluster-name=k8s"
+        - "--controllers=*,-cloud-node" # disable cloud-node controller
+        - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins
+        - "--leader-elect=true"
+        - "--route-reconciliation-period=10s"
+        - "--v=2"
+        - "--port=10267"
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: "4"
+          memory: 2Gi
+      livenessProbe:
+        httpGet:
+          path: /healthz
+          port: 10267
+        initialDelaySeconds: 20
+        periodSeconds: 10
+        timeoutSeconds: 5
+      volumeMounts:
         - name: etc-kubernetes
-          hostPath:
-            path: /etc/kubernetes
-        - name: cloud-config
-          secret:
-            secretName: azure-cloud-config
+          mountPath: /etc/kubernetes
         - name: etc-ssl
-          hostPath:
-            path: /etc/ssl
+          mountPath: /etc/ssl
+          readOnly: true
         - name: msi
-          hostPath:
-            path: /var/lib/waagent/ManagedIdentity-Settings
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: azure-cloud-config
-  namespace: kube-system
-type: Opaque
-stringData:
-  azure.json: |-
-    {
-      "cloud": "AzurePublicCloud",
-      "tenantId": "<tenant-id>",
-      "subscriptionId": "<subscription-id>",
-      "aadClientId": "<client-id>",
-      "aadClientSecret": "<client-secret>",
-      "resourceGroup": "<resource-group-name>",
-      "location": "<location>",
-      "vmType": "<vm-type>",
-      "subnetName": "<subnet-name>",
-      "securityGroupName": "<security-group-name>",
-      "vnetName": "<vnet-name>",
-      "vnetResourceGroup": "<vnet-resource-group>",
-      "routeTableName": "<route-table-name>",
-      "primaryAvailabilitySetName": "<primary-as-name>",
-      "primaryScaleSetName": "<primary-ss-name>",
-      "cloudProviderBackoff": true,
-      "cloudProviderBackoffRetries": 6,
-      "cloudProviderBackoffExponent": 1.5,
-      "cloudProviderBackoffDuration": 5,
-      "cloudProviderBackoffJitter": 1,
-      "cloudProviderRatelimit": true,
-      "cloudProviderRateLimitQPS": 6,
-      "cloudProviderRateLimitBucket": 20,
-      "useManagedIdentityExtension": false,
-      "userAssignedIdentityID": "",
-      "useInstanceMetadata": true,
-      "loadBalancerSku": "<loadbalancer-sku>",
-      "excludeMasterFromStandardLB": false,
-      "maximumLoadBalancerRuleCount": 250,
-      "enableMultipleStandardLoadBalancers": false,
-      "tags": "a=b,c=d"
-    }
+          mountPath: /var/lib/waagent/ManagedIdentity-Settings
+          readOnly: true
+  volumes:
+    - name: etc-kubernetes
+      hostPath:
+        path: /etc/kubernetes
+    - name: etc-ssl
+      hostPath:
+        path: /etc/ssl
+    - name: msi
+      hostPath:
+        path: /var/lib/waagent/ManagedIdentity-Settings
diff --git a/go.mod b/go.mod
index eb0dca0c4..a7d5ca529 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/onsi/gomega v1.18.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.7.0
 	golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
 	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
 	k8s.io/api v0.23.3
diff --git a/go.sum b/go.sum
index 9cb5f06f0..90189b783 100644
--- a/go.sum
+++ b/go.sum
@@ -446,9 +446,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
diff --git a/pkg/metrics/azure_metrics.go b/pkg/metrics/azure_metrics.go
index be2a7ae0c..a213d49f1 100644
--- a/pkg/metrics/azure_metrics.go
+++ b/pkg/metrics/azure_metrics.go
@@ -89,7 +89,7 @@ func (mc *MetricContext) Observe(rerr *retry.Error, labelAndValues ...interface{
 		attributes := append(mc.attributes, errorCode)
 		apiMetrics.errors.WithLabelValues(attributes...).Inc()
 	}
-	mc.logLatency(6, latency, append(labelAndValues, "error_code", rerr.ServiceErrorCode())...)
+	mc.logLatency(latency, append(labelAndValues, "error_code", rerr.ServiceErrorCode())...)
 }
 
 // ObserveOperationWithResult observes the request latency and failed requests of an operation.
@@ -101,15 +101,15 @@ func (mc *MetricContext) ObserveOperationWithResult(isOperationSucceeded bool, l
 		resultCode = "failed"
 		mc.CountFailedOperation()
 	}
-	mc.logLatency(3, latency, append(labelAndValues, "result_code", resultCode)...)
+	mc.logLatency(latency, append(labelAndValues, "result_code", resultCode)...)
 }
 
-func (mc *MetricContext) logLatency(logLevel int32, latency float64, additionalKeysAndValues ...interface{}) {
+func (mc *MetricContext) logLatency(latency float64, additionalKeysAndValues ...interface{}) {
 	keysAndValues := []interface{}{"latency_seconds", latency}
 	for i, label := range metricLabels {
 		keysAndValues = append(keysAndValues, label, mc.attributes[i])
 	}
-	klog.V(klog.Level(logLevel)).InfoS("Observed Request Latency", append(keysAndValues, additionalKeysAndValues...)...)
+	klog.V(3).InfoS("Observed Request Latency", append(keysAndValues, additionalKeysAndValues...)...)
 }
 
 // CountFailedOperation increase the number of failed operations
diff --git a/pkg/provider/azure_loadbalancer.go b/pkg/provider/azure_loadbalancer.go
index af2975802..5b6e038fe 100644
--- a/pkg/provider/azure_loadbalancer.go
+++ b/pkg/provider/azure_loadbalancer.go
@@ -259,9 +259,14 @@ func (az *Cloud) shouldChangeLoadBalancer(service *v1.Service, currLBName, clust
 		return false
 	}
 
-	// if the current LB is what we want, keep it
 	lbName := strings.TrimSuffix(currLBName, consts.InternalLoadBalancerNameSuffix)
+	// change the LB from vmSet dedicated to primary if the vmSet becomes the primary one
 	if strings.EqualFold(lbName, vmSetName) {
+		if lbName != clusterName &&
+			strings.EqualFold(az.VMSet.GetPrimaryVMSetName(), vmSetName) {
+			klog.V(2).Infof("shouldChangeLoadBalancer(%s, %s, %s): change the LB to another one", service.Name, currLBName, clusterName)
+			return true
+		}
 		return false
 	}
 	if strings.EqualFold(vmSetName, az.VMSet.GetPrimaryVMSetName()) && strings.EqualFold(clusterName, lbName) {
@@ -570,7 +575,8 @@ func (az *Cloud) getServiceLoadBalancer(service *v1.Service, clusterName string,
 		if isInternalLoadBalancer(&existingLB) != isInternal {
 			continue
 		}
-		status, fipConfig, err := az.getServiceLoadBalancerStatus(service, &existingLB)
+		var fipConfig *network.FrontendIPConfiguration
+		status, fipConfig, err = az.getServiceLoadBalancerStatus(service, &existingLB)
 		if err != nil {
 			return nil, nil, false, err
 		}
@@ -578,6 +584,7 @@ func (az *Cloud) getServiceLoadBalancer(service *v1.Service, clusterName string,
 			// service is not on this load balancer
 			continue
 		}
+		klog.V(4).Infof("getServiceLoadBalancer(%s, %s, %v): current lb ip: %s", service.Name, clusterName, wantLb, status.Ingress[0].IP)
 
 		// select another load balancer instead of returning
 		// the current one if the change is needed
@@ -603,7 +610,7 @@ func (az *Cloud) getServiceLoadBalancer(service *v1.Service, clusterName string,
 			return nil, nil, false, err
 		}
 
-		return selectedLB, nil, exists, err
+		return selectedLB, status, exists, err
 	}
 
 	// create a default LB with meta data if not present
@@ -1405,11 +1412,12 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 		return nil, err
 	}
 
-	lb, _, _, err := az.getServiceLoadBalancer(service, clusterName, nodes, wantLb, existingLBs)
+	lb, lbStatus, _, err := az.getServiceLoadBalancer(service, clusterName, nodes, wantLb, existingLBs)
 	if err != nil {
 		klog.Errorf("reconcileLoadBalancer: failed to get load balancer for service %q, error: %v", serviceName, err)
 		return nil, err
 	}
+
 	lbName := *lb.Name
 	lbResourceGroup := az.getLoadBalancerResourceGroup()
 	lbBackendPoolID := az.getBackendPoolID(lbName, az.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, service))
@@ -1431,7 +1439,7 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
 	}
 
 	// reconcile the load balancer's frontend IP configurations.
-	ownedFIPConfig, changed, err := az.reconcileFrontendIPConfigs(clusterName, service, lb, wantLb, defaultLBFrontendIPConfigName)
+	ownedFIPConfig, changed, err := az.reconcileFrontendIPConfigs(clusterName, service, lb, lbStatus, wantLb, defaultLBFrontendIPConfigName)
 	if err != nil {
 		return lb, err
 	}
@@ -1623,7 +1631,7 @@ func (az *Cloud) reconcileLBRules(lb *network.LoadBalancer, service *v1.Service,
 	return dirtyRules
 }
 
-func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Service, lb *network.LoadBalancer, wantLb bool, defaultLBFrontendIPConfigName string) (*network.FrontendIPConfiguration, bool, error) {
+func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Service, lb *network.LoadBalancer, status *v1.LoadBalancerStatus, wantLb bool, defaultLBFrontendIPConfigName string) (*network.FrontendIPConfiguration, bool, error) {
 	var err error
 	lbName := *lb.Name
 	serviceName := getServiceName(service)
@@ -1723,8 +1731,13 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv
 				if loadBalancerIP != "" {
 					configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic
 					configProperties.PrivateIPAddress = &loadBalancerIP
+				} else if status != nil && len(status.Ingress) > 0 {
+					klog.V(4).Infof("reconcileFrontendIPConfigs for service (%s): keep the original private IP %s", serviceName, status.Ingress[0].IP)
+					configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic
+					configProperties.PrivateIPAddress = to.StringPtr(status.Ingress[0].IP)
 				} else {
 					// We'll need to call GetLoadBalancer later to retrieve allocated IP.
+					klog.V(4).Infof("reconcileFrontendIPConfigs for service (%s): dynamically allocate the private IP", serviceName)
 					configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodDynamic
 				}
 
diff --git a/site/content/en/install/azure-ccm.md b/site/content/en/install/azure-ccm.md
index fdefd9250..95a659050 100644
--- a/site/content/en/install/azure-ccm.md
+++ b/site/content/en/install/azure-ccm.md
@@ -22,7 +22,7 @@ To deploy Azure cloud controller manager, the following components need to be co
 |Flag|Value|Remark|
 |----|-----|------|
 |`--cloud-provider`|external|cloud-provider should be set external|
-|`--azure-container-registry-config`|/etc/kubernetes/cloud-config/azure.json|Used for Azure credential provider|
+|`--azure-container-registry-config`|/etc/kubernetes/azure.json|Used for Azure credential provider|
 
 ### kube-controller-manager
 
@@ -39,12 +39,10 @@ Do not set flag `--cloud-provider`.
 
 ### azure-cloud-controller-manager
 
-azure-cloud-controller-manager should be run as Deployment with multiple replicas or Kubelet static Pods on each master Node.
-
 |Flag|Value|Remark|
 |---|---|---|
 |`--cloud-provider`|azure|cloud-provider should be set azure|
-|`--cloud-config`|/etc/kubernetes/cloud-config/azure.json|Path for [cloud provider config](../configs.md)|
+|`--cloud-config`|/etc/kubernetes/azure.json|Path for [cloud provider config](../configs)|
 |`--controllers`|*,-cloud-node | cloud node controller should be disabled|
 |`--configure-cloud-routes`| "false" for Azure CNI and "true" for other network plugins| Used for non-AzureCNI clusters |
 
diff --git a/site/content/en/install/configs.md b/site/content/en/install/configs.md
index ad1f3e239..e4d8c0278 100644
--- a/site/content/en/install/configs.md
+++ b/site/content/en/install/configs.md
@@ -225,7 +225,7 @@ Here is an example of per-client config:
 
 When running Kubelet with kube-controller-manager, it also supports running without Azure identity since v1.15.0.
 
-Both kube-controller-manager and kubelet should configure `--cloud-provider=azure --cloud-config=/etc/kubernetes/cloud-config/azure.json`, but the contents for `azure.json` are different:
+Both kube-controller-manager and kubelet should configure `--cloud-provider=azure --cloud-config=/etc/kubernetes/azure.json`, but the contents for `azure.json` are different:
 
 (1) For kube-controller-manager, refer the above part for setting `azure.json`.
 
diff --git a/site/content/en/topics/availability-zones.md b/site/content/en/topics/availability-zones.md
index f162cddc4..7ca339e3f 100644
--- a/site/content/en/topics/availability-zones.md
+++ b/site/content/en/topics/availability-zones.md
@@ -29,7 +29,7 @@ Kubernetes v1.12 adds support for [Azure availability zones (AZ)](https://azure.
 
 ## Pre-requirements
 
-Because only standard load balancer is supported with AZ, it is a prerequisite to enable AZ for the cluster. It should be configured in Azure cloud provider configure file (e.g. `/etc/kubernetes/cloud-config/azure.json`):
+Because only standard load balancer is supported with AZ, it is a prerequisite to enable AZ for the cluster. It should be configured in Azure cloud provider configure file (e.g. `/etc/kubernetes/azure.json`):
 
 ```json
 {
diff --git a/site/content/en/topics/credential-provider.md b/site/content/en/topics/credential-provider.md
index f81213c35..59867e088 100644
--- a/site/content/en/topics/credential-provider.md
+++ b/site/content/en/topics/credential-provider.md
@@ -7,7 +7,7 @@ description: >
     Detailed steps to setup out-of-tree Kubelet Credential Provider.
 ---
 
-> Note: The Kubelet credential provider feature is still in alpha and shouldn't be used in production environments. Please use `--azure-container-registry-config=/etc/kubernetes/cloud-config/azure.json` if you need pulling images from ACR in production.
+> Note: The Kubelet credential provider feature is still in alpha and shouldn't be used in production environments. Please use `--azure-container-registry-config=/etc/kubernetes/azure.json` if you need pulling images from ACR in production.
 
 As part of [Out-of-Tree Credential Providers]([enhancements/keps/sig-cloud-provider/2133-out-of-tree-credential-provider at master · kubernetes/enhancements (github.com)](https://github.com/kubernetes/enhancements/tree/master/keps/sig-cloud-provider/2133-out-of-tree-credential-provider)), the kubelet builtin image pulling from ACR (which could be enabled by setting `kubelet --azure-container-registry-config=<config-file>`) would be moved out-of-tree credential plugin `acr-credential-provider`. Please refer the original [KEP](https://github.com/kubernetes/enhancements/tree/master/keps/sig-cloud-provider/2133-out-of-tree-credential-provider) for details.
 
@@ -33,6 +33,6 @@ providers:
   - "*.azurecr.us"
   - "*.azurecr.*" # Only required for custom Azure cloud.
   args:
-  - /etc/kubernetes/cloud-config/azure.json
+  - /etc/kubernetes/azure.json
 ```
 
diff --git a/site/content/en/topics/cross-resource-group-nodes.md b/site/content/en/topics/cross-resource-group-nodes.md
index 5eea1d536..9c79677df 100644
--- a/site/content/en/topics/cross-resource-group-nodes.md
+++ b/site/content/en/topics/cross-resource-group-nodes.md
@@ -48,7 +48,7 @@ For example,
 ```shell script
 kubelet ... \
   --cloud-provider=azure \
-  --cloud-config=/etc/kubernetes/cloud-config/azure.json \
+  --cloud-config=/etc/kubernetes/azure.json \
   --node-labels=node.kubernetes.io/exclude-balancer=true,kubernetes.azure.com/resource-group=<rg-name>
 ```
 
diff --git a/site/content/en/topics/loadbalancer.md b/site/content/en/topics/loadbalancer.md
index 8b55ec3bc..6ad49849a 100644
--- a/site/content/en/topics/loadbalancer.md
+++ b/site/content/en/topics/loadbalancer.md
@@ -28,9 +28,11 @@ Below is a list of annotations supported for Kubernetes services with type `Load
 | `service.beta.kubernetes.io/azure-allowed-service-tags`      | List of allowed service tags | Specify a list of allowed [service tags](https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#service-tags) separated by comma. | v1.11.0 and later |
 | `service.beta.kubernetes.io/azure-load-balancer-tcp-idle-timeout` | TCP idle timeouts in minutes | Specify the time, in minutes, for TCP connection idle timeouts to occur on the load balancer. Default and minimum value is 4. Maximum value is 30. Must be an integer. |  v1.11.4, v1.12.0 and later |
 | `service.beta.kubernetes.io/azure-pip-name` | Name of PIP | Specify the PIP that will be applied to load balancer | v1.16 and later |
+| `service.beta.kubernetes.io/azure-load-balancer-disable-tcp-reset` | `true` | Disable `enableTcpReset` for SLB | v1.16-v1.18. The annotation has been deprecated and would be removed in a future release. |
 | `service.beta.kubernetes.io/azure-pip-tags` | Tags of the PIP | Specify the tags of the PIP that will be associated to the load balancer typed service. [Doc](../tagging-resources) | v1.20 and later |
 | `service.beta.kubernetes.io/azure-load-balancer-health-probe-interval` | Health probe interval | Refer to the detailed docs [here](#custom-load-balancer-health-probe) | v1.21 and later  |
 | `service.beta.kubernetes.io/azure-load-balancer-health-probe-num-of-probe` | The minimum number of unhealthy responses of health probe  |  Refer to the detailed docs [here](#custom-load-balancer-health-probe) |	v1.21 and later |
+| `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` | Health probe protocol of the load balancer typed service | Refer to the detailed docs [here](#custom-load-balancer-health-probe) | v1.20 and later |
 | `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` | Request path of the health probe | Refer to the detailed docs [here](#custom-load-balancer-health-probe) | v1.20 and later |
 | `service.beta.kubernetes.io/port_{port}_health-probe_interval` | Health probe interval |  {port} is port number of service.  Refer to the detailed docs [here](#custom-load-balancer-health-probe) | v1.21 and later  |
 | `service.beta.kubernetes.io/port_{port}_health-probe_num-of-probe` | The minimum number of unhealthy responses of health probe  | {port} is port number of service. Refer to the detailed docs [here](#custom-load-balancer-health-probe) |	v1.21 and later |
@@ -160,9 +162,9 @@ Currently, the default protocol of the health probe varies among services with d
 1. for cluster TCP services, TCP would be used.
 1. for cluster UDP services, no health probes.
 
-Since v1.20, two service annotations `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` are introduced, which determine the new health probe behavior. If the spec.ports.appProtocol is set, both local and cluster TCP services would use the specified health probe protocol. If the `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` is set, the specified request path would be used instead of `/healthz`. Note that the request path would be ignored when using TCP or the spec.ports.appProtocol is empty. More specifically:
+Since v1.20, two service annotations `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` and `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` are introduced, which determine the new health probe behavior. If the `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` is set, both local and cluster TCP services would use the specified health probe protocol. If the `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` is set, the specified request path would be used instead of `/healthz`. Note that the request path would be ignored when using TCP or the `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` is empty. More specifically:
 
-| `externalTrafficPolicy` | spec.ports.AppProtocol | `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` | protocol | request path |
+| `externalTrafficPolicy` | `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol` | `service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path` | protocol | request path |
 | ------------------------------------------------------------ | ---------------------------- | ------------------------------------------------------------ |------| ----- |
 | local |  | (ignored) | http | `/healthz` |
 | local | tcp | (ignored) | tcp | null |
@@ -180,6 +182,8 @@ Since v1.21, two service annotations `service.beta.kubernetes.io/azure-load-bala
 
 Because [MixedProtocolLBService](https://kubernetes.io/docs/concepts/services-networking/service/#load-balancers-with-mixed-protocol-types) feature is in alpha stage, Ports in one service may have different probe configurations. Following annotations are introduced to customize probe configuration for one port.
 
+[AppProtocol field in port becomes GA in kubernetes 1.20.](https://kubernetes.io/docs/concepts/services-networking/service/#application-protocol) The value of AppProtocol in port will overwrite the one defined in `service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol`
+
 | port specific annotation | global probe annotation | 
 | --| -- | 
 |service.beta.kubernetes.io/port_{port}_health-probe_request-path|service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path|
@@ -194,6 +198,7 @@ kind: Service
 metadata:
   name: appservice
   annotations:
+    service.beta.kubernetes.io/azure-load-balancer-health-probe-protocol: "http"
     service.beta.kubernetes.io/azure-load-balancer-health-probe-num-of-probe: "5"
     service.beta.kubernetes.io/port_443_health-probe_num-of-probe: "4"
 spec:
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 9b0f5eccc..65385a78d 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -255,7 +255,7 @@ github.com/spf13/cobra
 # github.com/spf13/pflag v1.0.5
 ## explicit; go 1.12
 github.com/spf13/pflag
-# github.com/stretchr/testify v1.7.1
+# github.com/stretchr/testify v1.7.0
 ## explicit; go 1.13
 github.com/stretchr/testify/assert
 # go.etcd.io/etcd/api/v3 v3.5.1


Test patch:
diff --git a/e2e.Dockerfile b/e2e.Dockerfile
index e5d21a340..d84c85517 100644
--- a/e2e.Dockerfile
+++ b/e2e.Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.18.0-buster
+FROM golang:1.17-buster
 
 WORKDIR /go/src/sigs.k8s.io/cloud-provider-azure
 
diff --git a/pkg/provider/azure_loadbalancer_test.go b/pkg/provider/azure_loadbalancer_test.go
index 7ec14dcb3..5a5b5c6de 100644
--- a/pkg/provider/azure_loadbalancer_test.go
+++ b/pkg/provider/azure_loadbalancer_test.go
@@ -1319,6 +1319,37 @@ func TestGetServiceLoadBalancer(t *testing.T) {
 			expectedExists: false,
 			expectedError:  false,
 		},
+		{
+			desc:    "getServiceLoadBalancer should create a new lb and return the status of the previous lb",
+			sku:     "Basic",
+			wantLB:  true,
+			service: getTestService("service1", v1.ProtocolTCP, map[string]string{consts.ServiceAnnotationLoadBalancerMode: "as", consts.ServiceAnnotationLoadBalancerInternal: consts.TrueAnnotationValue}, false, 80),
+			existingLBs: []network.LoadBalancer{
+				{
+					Name:     to.StringPtr("as-internal"),
+					Location: to.StringPtr("westus"),
+					LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{
+						FrontendIPConfigurations: &[]network.FrontendIPConfiguration{
+							{
+								Name: to.StringPtr("aservice1"),
+								FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
+									PrivateIPAddress: to.StringPtr("1.2.3.4"),
+								},
+							},
+						},
+					},
+				},
+			},
+			expectedLB: &network.LoadBalancer{
+				Name:     to.StringPtr("testCluster-internal"),
+				Location: to.StringPtr("westus"),
+				Sku: &network.LoadBalancerSku{
+					Name: network.LoadBalancerSkuNameBasic,
+				},
+				LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{},
+			},
+			expectedStatus: &v1.LoadBalancerStatus{Ingress: []v1.LoadBalancerIngress{{IP: "1.2.3.4", Hostname: ""}}},
+		},
 	}
 
 	ctrl := gomock.NewController(t)
@@ -1331,6 +1362,7 @@ func TestGetServiceLoadBalancer(t *testing.T) {
 		mockLBsClient := mockloadbalancerclient.NewMockInterface(ctrl)
 		mockLBsClient.EXPECT().CreateOrUpdate(gomock.Any(), "rg", gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 		mockLBsClient.EXPECT().List(gomock.Any(), "rg").Return(test.existingLBs, nil)
+		mockLBsClient.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 		az.LoadBalancerClient = mockLBsClient
 
 		for _, existingLB := range test.existingLBs {
@@ -1339,7 +1371,9 @@ func TestGetServiceLoadBalancer(t *testing.T) {
 				t.Fatalf("TestCase[%d] meets unexpected error: %v", i, err)
 			}
 		}
-		test.service.Annotations = test.annotations
+		if test.annotations != nil {
+			test.service.Annotations = test.annotations
+		}
 		az.LoadBalancerSku = test.sku
 		lb, status, exists, err := az.getServiceLoadBalancer(&test.service, testClusterName,
 			clusterResources.nodes, test.wantLB, []network.LoadBalancer{})
@@ -4408,6 +4442,18 @@ func TestShouldChangeLoadBalancer(t *testing.T) {
 		res := cloud.shouldChangeLoadBalancer(&service, "testCluster-internal", "testCluster")
 		assert.False(t, res)
 	})
+
+	t.Run("shouldChangeLoadBalancer should return true if the mode is the same as the current LB but the vmSet is the primary one", func(t *testing.T) {
+		cloud.LoadBalancerSku = consts.LoadBalancerSkuStandard
+		cloud.EnableMultipleStandardLoadBalancers = true
+		cloud.PrimaryAvailabilitySetName = "vmss-1"
+		annotations := map[string]string{
+			consts.ServiceAnnotationLoadBalancerMode: "vmss-1",
+		}
+		service := getTestService("service1", v1.ProtocolTCP, annotations, false, 80)
+		res := cloud.shouldChangeLoadBalancer(&service, "vmss-1", "testCluster")
+		assert.True(t, res)
+	})
 }
 
 func TestRemoveFrontendIPConfigurationFromLoadBalancerDelete(t *testing.T) {
@@ -4509,11 +4555,13 @@ func TestReconcileZonesForFrontendIPConfigs(t *testing.T) {
 		service                   v1.Service
 		existingFrontendIPConfigs []network.FrontendIPConfiguration
 		existingPIP               network.PublicIPAddress
+		status                    *v1.LoadBalancerStatus
 		getPIPError               *retry.Error
 		getZoneError              *retry.Error
 		regionZonesMap            map[string][]string
 		expectedZones             *[]string
 		expectedDirty             bool
+		expectedIP                string
 		expectedErr               error
 	}{
 		{
@@ -4577,6 +4625,17 @@ func TestReconcileZonesForFrontendIPConfigs(t *testing.T) {
 			expectedZones: &[]string{"1"},
 			expectedDirty: true,
 		},
+		{
+			description: "reconcileFrontendIPConfigs should reuse the existing private IP for internal services",
+			service:     getInternalTestService("test", 80),
+			status: &v1.LoadBalancerStatus{
+				Ingress: []v1.LoadBalancerIngress{
+					{IP: "1.2.3.4"},
+				},
+			},
+			expectedIP:    "1.2.3.4",
+			expectedDirty: true,
+		},
 	} {
 		t.Run(tc.description, func(t *testing.T) {
 			cloud := GetTestCloud(ctrl)
@@ -4599,7 +4658,7 @@ func TestReconcileZonesForFrontendIPConfigs(t *testing.T) {
 			cloud.ZoneClient = zoneClient
 
 			defaultLBFrontendIPConfigName := cloud.getDefaultFrontendIPConfigName(&tc.service)
-			_, dirty, err := cloud.reconcileFrontendIPConfigs("testCluster", &tc.service, &lb, true, defaultLBFrontendIPConfigName)
+			_, dirty, err := cloud.reconcileFrontendIPConfigs("testCluster", &tc.service, &lb, tc.status, true, defaultLBFrontendIPConfigName)
 			if tc.expectedErr == nil {
 				assert.NoError(t, err)
 			} else {
@@ -4612,6 +4671,11 @@ func TestReconcileZonesForFrontendIPConfigs(t *testing.T) {
 					assert.Equal(t, tc.expectedZones, fip.Zones)
 				}
 			}
+
+			if tc.expectedIP != "" {
+				assert.Equal(t, network.IPAllocationMethodStatic, (*lb.FrontendIPConfigurations)[0].PrivateIPAllocationMethod)
+				assert.Equal(t, tc.expectedIP, to.String((*lb.FrontendIPConfigurations)[0].PrivateIPAddress))
+			}
 		})
 	}
 }
diff --git a/tests/k8s-azure/manifest/cluster-api/vmss-multi-nodepool.yaml b/tests/k8s-azure/manifest/cluster-api/vmss-multi-nodepool.yaml
index c05765fe7..048b991dd 100644
--- a/tests/k8s-azure/manifest/cluster-api/vmss-multi-nodepool.yaml
+++ b/tests/k8s-azure/manifest/cluster-api/vmss-multi-nodepool.yaml
@@ -454,7 +454,7 @@ data:
         name: cloud-controller-manager
     ---
     apiVersion: v1
-    kind: Deployment
+    kind: Pod
     metadata:
       name: cloud-controller-manager
       namespace: kube-system
@@ -462,76 +462,65 @@ data:
         tier: control-plane
         component: cloud-controller-manager
     spec:
-      replicas: 1
-      selector:
-        matchLabels:
-          tier: control-plane
-          component: cloud-controller-manager
-    template:
-      metadata:
-        labels:
-          tier: control-plane
-          component: cloud-controller-manager
-      spec:
-        priorityClassName: system-node-critical
-        hostNetwork: true
-        nodeSelector:
-          node-role.kubernetes.io/master: ""
-        serviceAccountName: cloud-controller-manager
-        tolerations:
-          - key: node-role.kubernetes.io/master
-            effect: NoSchedule
-        containers:
-          - name: cloud-controller-manager
-            image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG}
-            imagePullPolicy: IfNotPresent
-            command: ["cloud-controller-manager"]
-            args:
-              - "--allocate-node-cidrs=true"
-              - "--enable-dynamic-reloading=true"
-              - "--cloud-config=/etc/kubernetes/azure.json"
-              - "--cloud-provider=azure"
-              - "--cluster-cidr=10.244.0.0/16"
-              - "--cluster-name=${CLUSTER_NAME}"
-              - "--controllers=*,-cloud-node" # disable cloud-node controller
-              - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins
-              - "--leader-elect=true"
-              - "--route-reconciliation-period=10s"
-              - "--v=4"
-              - "--port=10267"
-            resources:
-              requests:
-                cpu: 100m
-                memory: 128Mi
-              limits:
-                cpu: "4"
-                memory: 2Gi
-            livenessProbe:
-              httpGet:
-                path: /healthz
-                port: 10267
-              initialDelaySeconds: 20
-              periodSeconds: 10
-              timeoutSeconds: 5
-            volumeMounts:
-              - name: etc-kubernetes
-                mountPath: /etc/kubernetes
-              - name: etc-ssl
-                mountPath: /etc/ssl
-                readOnly: true
-              - name: msi
-                mountPath: /var/lib/waagent/ManagedIdentity-Settings
-                readOnly: true
-        volumes:
-          - name: etc-kubernetes
-            hostPath:
-              path: /etc/kubernetes
-          - name: etc-ssl
-            hostPath:
-              path: /etc/ssl
-          - name: msi
-            hostPath:
-              path: /var/lib/waagent/ManagedIdentity-Settings
+      priorityClassName: system-node-critical
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      serviceAccountName: cloud-controller-manager
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      containers:
+        - name: cloud-controller-manager
+          image: ${AZURE_CLOUD_CONTROLLER_MANAGER_IMG}
+          imagePullPolicy: IfNotPresent
+          command: ["cloud-controller-manager"]
+          args:
+            - "--allocate-node-cidrs=true"
+            - "--enable-dynamic-reloading=true"
+            - "--cloud-config=/etc/kubernetes/azure.json"
+            - "--cloud-provider=azure"
+            - "--cluster-cidr=10.244.0.0/16"
+            - "--cluster-name=${CLUSTER_NAME}"
+            - "--controllers=*,-cloud-node" # disable cloud-node controller
+            - "--configure-cloud-routes=true" # "false" for Azure CNI and "true" for other network plugins
+            - "--leader-elect=true"
+            - "--route-reconciliation-period=10s"
+            - "--v=4"
+            - "--port=10267"
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: "4"
+              memory: 2Gi
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 10267
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: etc-kubernetes
+              mountPath: /etc/kubernetes
+            - name: etc-ssl
+              mountPath: /etc/ssl
+              readOnly: true
+            - name: msi
+              mountPath: /var/lib/waagent/ManagedIdentity-Settings
+              readOnly: true
+      volumes:
+        - name: etc-kubernetes
+          hostPath:
+            path: /etc/kubernetes
+        - name: etc-ssl
+          hostPath:
+            path: /etc/ssl
+        - name: msi
+          hostPath:
+            path: /var/lib/waagent/ManagedIdentity-Settings
 kind: ConfigMap
 metadata:
   annotations:
diff --git a/vendor/github.com/stretchr/testify/assert/assertion_compare.go b/vendor/github.com/stretchr/testify/assert/assertion_compare.go
index 3bb22a971..41649d267 100644
--- a/vendor/github.com/stretchr/testify/assert/assertion_compare.go
+++ b/vendor/github.com/stretchr/testify/assert/assertion_compare.go
@@ -3,7 +3,6 @@ package assert
 import (
 	"fmt"
 	"reflect"
-	"time"
 )
 
 type CompareType int
@@ -31,8 +30,6 @@ var (
 	float64Type = reflect.TypeOf(float64(1))
 
 	stringType = reflect.TypeOf("")
-
-	timeType = reflect.TypeOf(time.Time{})
 )
 
 func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
@@ -302,27 +299,6 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 				return compareLess, true
 			}
 		}
-	// Check for known struct types we can check for compare results.
-	case reflect.Struct:
-		{
-			// All structs enter here. We're not interested in most types.
-			if !canConvert(obj1Value, timeType) {
-				break
-			}
-
-			// time.Time can compared!
-			timeObj1, ok := obj1.(time.Time)
-			if !ok {
-				timeObj1 = obj1Value.Convert(timeType).Interface().(time.Time)
-			}
-
-			timeObj2, ok := obj2.(time.Time)
-			if !ok {
-				timeObj2 = obj2Value.Convert(timeType).Interface().(time.Time)
-			}
-
-			return compare(timeObj1.UnixNano(), timeObj2.UnixNano(), reflect.Int64)
-		}
 	}
 
 	return compareEqual, false
@@ -334,10 +310,7 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) {
 //    assert.Greater(t, float64(2), float64(1))
 //    assert.Greater(t, "b", "a")
 func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
-	return compareTwoValues(t, e1, e2, []CompareType{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs...)
+	return compareTwoValues(t, e1, e2, []CompareType{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs)
 }
 
 // GreaterOrEqual asserts that the first element is greater than or equal to the second
@@ -347,10 +320,7 @@ func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface
 //    assert.GreaterOrEqual(t, "b", "a")
 //    assert.GreaterOrEqual(t, "b", "b")
 func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
-	return compareTwoValues(t, e1, e2, []CompareType{compareGreater, compareEqual}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs...)
+	return compareTwoValues(t, e1, e2, []CompareType{compareGreater, compareEqual}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs)
 }
 
 // Less asserts that the first element is less than the second
@@ -359,10 +329,7 @@ func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...in
 //    assert.Less(t, float64(1), float64(2))
 //    assert.Less(t, "a", "b")
 func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
-	return compareTwoValues(t, e1, e2, []CompareType{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs...)
+	return compareTwoValues(t, e1, e2, []CompareType{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs)
 }
 
 // LessOrEqual asserts that the first element is less than or equal to the second
@@ -372,10 +339,7 @@ func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{})
 //    assert.LessOrEqual(t, "a", "b")
 //    assert.LessOrEqual(t, "b", "b")
 func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
-	return compareTwoValues(t, e1, e2, []CompareType{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs...)
+	return compareTwoValues(t, e1, e2, []CompareType{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs)
 }
 
 // Positive asserts that the specified element is positive
@@ -383,11 +347,8 @@ func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...inter
 //    assert.Positive(t, 1)
 //    assert.Positive(t, 1.23)
 func Positive(t TestingT, e interface{}, msgAndArgs ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
 	zero := reflect.Zero(reflect.TypeOf(e))
-	return compareTwoValues(t, e, zero.Interface(), []CompareType{compareGreater}, "\"%v\" is not positive", msgAndArgs...)
+	return compareTwoValues(t, e, zero.Interface(), []CompareType{compareGreater}, "\"%v\" is not positive", msgAndArgs)
 }
 
 // Negative asserts that the specified element is negative
@@ -395,11 +356,8 @@ func Positive(t TestingT, e interface{}, msgAndArgs ...interface{}) bool {
 //    assert.Negative(t, -1)
 //    assert.Negative(t, -1.23)
 func Negative(t TestingT, e interface{}, msgAndArgs ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
 	zero := reflect.Zero(reflect.TypeOf(e))
-	return compareTwoValues(t, e, zero.Interface(), []CompareType{compareLess}, "\"%v\" is not negative", msgAndArgs...)
+	return compareTwoValues(t, e, zero.Interface(), []CompareType{compareLess}, "\"%v\" is not negative", msgAndArgs)
 }
 
 func compareTwoValues(t TestingT, e1 interface{}, e2 interface{}, allowedComparesResults []CompareType, failMessage string, msgAndArgs ...interface{}) bool {
diff --git a/vendor/github.com/stretchr/testify/assert/assertion_compare_can_convert.go b/vendor/github.com/stretchr/testify/assert/assertion_compare_can_convert.go
deleted file mode 100644
index df22c47fc..000000000
--- a/vendor/github.com/stretchr/testify/assert/assertion_compare_can_convert.go
+++ /dev/null
@@ -1,16 +0,0 @@
-//go:build go1.17
-// +build go1.17
-
-// TODO: once support for Go 1.16 is dropped, this file can be
-//       merged/removed with assertion_compare_go1.17_test.go and
-//       assertion_compare_legacy.go
-
-package assert
-
-import "reflect"
-
-// Wrapper around reflect.Value.CanConvert, for compatability
-// reasons.
-func canConvert(value reflect.Value, to reflect.Type) bool {
-	return value.CanConvert(to)
-}
diff --git a/vendor/github.com/stretchr/testify/assert/assertion_compare_legacy.go b/vendor/github.com/stretchr/testify/assert/assertion_compare_legacy.go
deleted file mode 100644
index 1701af2a3..000000000
--- a/vendor/github.com/stretchr/testify/assert/assertion_compare_legacy.go
+++ /dev/null
@@ -1,16 +0,0 @@
-//go:build !go1.17
-// +build !go1.17
-
-// TODO: once support for Go 1.16 is dropped, this file can be
-//       merged/removed with assertion_compare_go1.17_test.go and
-//       assertion_compare_can_convert.go
-
-package assert
-
-import "reflect"
-
-// Older versions of Go does not have the reflect.Value.CanConvert
-// method.
-func canConvert(value reflect.Value, to reflect.Type) bool {
-	return false
-}
diff --git a/vendor/github.com/stretchr/testify/assert/assertion_format.go b/vendor/github.com/stretchr/testify/assert/assertion_format.go
index 27e2420ed..4dfd1229a 100644
--- a/vendor/github.com/stretchr/testify/assert/assertion_format.go
+++ b/vendor/github.com/stretchr/testify/assert/assertion_format.go
@@ -123,18 +123,6 @@ func ErrorAsf(t TestingT, err error, target interface{}, msg string, args ...int
 	return ErrorAs(t, err, target, append([]interface{}{msg}, args...)...)
 }
 
-// ErrorContainsf asserts that a function returned an error (i.e. not `nil`)
-// and that the error contains the specified substring.
-//
-//   actualObj, err := SomeFunction()
-//   assert.ErrorContainsf(t, err,  expectedErrorSubString, "error message %s", "formatted")
-func ErrorContainsf(t TestingT, theError error, contains string, msg string, args ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
-	return ErrorContains(t, theError, contains, append([]interface{}{msg}, args...)...)
-}
-
 // ErrorIsf asserts that at least one of the errors in err's chain matches target.
 // This is a wrapper for errors.Is.
 func ErrorIsf(t TestingT, err error, target error, msg string, args ...interface{}) bool {
diff --git a/vendor/github.com/stretchr/testify/assert/assertion_forward.go b/vendor/github.com/stretchr/testify/assert/assertion_forward.go
index d9ea368d0..25337a6f0 100644
--- a/vendor/github.com/stretchr/testify/assert/assertion_forward.go
+++ b/vendor/github.com/stretchr/testify/assert/assertion_forward.go
@@ -222,30 +222,6 @@ func (a *Assertions) ErrorAsf(err error, target interface{}, msg string, args ..
 	return ErrorAsf(a.t, err, target, msg, args...)
 }
 
-// ErrorContains asserts that a function returned an error (i.e. not `nil`)
-// and that the error contains the specified substring.
-//
-//   actualObj, err := SomeFunction()
-//   a.ErrorContains(err,  expectedErrorSubString)
-func (a *Assertions) ErrorContains(theError error, contains string, msgAndArgs ...interface{}) bool {
-	if h, ok := a.t.(tHelper); ok {
-		h.Helper()
-	}
-	return ErrorContains(a.t, theError, contains, msgAndArgs...)
-}
-
-// ErrorContainsf asserts that a function returned an error (i.e. not `nil`)
-// and that the error contains the specified substring.
-//
-//   actualObj, err := SomeFunction()
-//   a.ErrorContainsf(err,  expectedErrorSubString, "error message %s", "formatted")
-func (a *Assertions) ErrorContainsf(theError error, contains string, msg string, args ...interface{}) bool {
-	if h, ok := a.t.(tHelper); ok {
-		h.Helper()
-	}
-	return ErrorContainsf(a.t, theError, contains, msg, args...)
-}
-
 // ErrorIs asserts that at least one of the errors in err's chain matches target.
 // This is a wrapper for errors.Is.
 func (a *Assertions) ErrorIs(err error, target error, msgAndArgs ...interface{}) bool {
diff --git a/vendor/github.com/stretchr/testify/assert/assertion_order.go b/vendor/github.com/stretchr/testify/assert/assertion_order.go
index 759448783..1c3b47182 100644
--- a/vendor/github.com/stretchr/testify/assert/assertion_order.go
+++ b/vendor/github.com/stretchr/testify/assert/assertion_order.go
@@ -50,7 +50,7 @@ func isOrdered(t TestingT, object interface{}, allowedComparesResults []CompareT
 //    assert.IsIncreasing(t, []float{1, 2})
 //    assert.IsIncreasing(t, []string{"a", "b"})
 func IsIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
-	return isOrdered(t, object, []CompareType{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs...)
+	return isOrdered(t, object, []CompareType{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs)
 }
 
 // IsNonIncreasing asserts that the collection is not increasing
@@ -59,7 +59,7 @@ func IsIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) boo
 //    assert.IsNonIncreasing(t, []float{2, 1})
 //    assert.IsNonIncreasing(t, []string{"b", "a"})
 func IsNonIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
-	return isOrdered(t, object, []CompareType{compareEqual, compareGreater}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs...)
+	return isOrdered(t, object, []CompareType{compareEqual, compareGreater}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs)
 }
 
 // IsDecreasing asserts that the collection is decreasing
@@ -68,7 +68,7 @@ func IsNonIncreasing(t TestingT, object interface{}, msgAndArgs ...interface{})
 //    assert.IsDecreasing(t, []float{2, 1})
 //    assert.IsDecreasing(t, []string{"b", "a"})
 func IsDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
-	return isOrdered(t, object, []CompareType{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs...)
+	return isOrdered(t, object, []CompareType{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs)
 }
 
 // IsNonDecreasing asserts that the collection is not decreasing
@@ -77,5 +77,5 @@ func IsDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) boo
 //    assert.IsNonDecreasing(t, []float{1, 2})
 //    assert.IsNonDecreasing(t, []string{"a", "b"})
 func IsNonDecreasing(t TestingT, object interface{}, msgAndArgs ...interface{}) bool {
-	return isOrdered(t, object, []CompareType{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs...)
+	return isOrdered(t, object, []CompareType{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs)
 }
diff --git a/vendor/github.com/stretchr/testify/assert/assertions.go b/vendor/github.com/stretchr/testify/assert/assertions.go
index 0357b2231..bcac4401f 100644
--- a/vendor/github.com/stretchr/testify/assert/assertions.go
+++ b/vendor/github.com/stretchr/testify/assert/assertions.go
@@ -718,14 +718,10 @@ func NotEqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...inte
 // return (false, false) if impossible.
 // return (true, false) if element was not found.
 // return (true, true) if element was found.
-func containsElement(list interface{}, element interface{}) (ok, found bool) {
+func includeElement(list interface{}, element interface{}) (ok, found bool) {
 
 	listValue := reflect.ValueOf(list)
-	listType := reflect.TypeOf(list)
-	if listType == nil {
-		return false, false
-	}
-	listKind := listType.Kind()
+	listKind := reflect.TypeOf(list).Kind()
 	defer func() {
 		if e := recover(); e != nil {
 			ok = false
@@ -768,7 +764,7 @@ func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bo
 		h.Helper()
 	}
 
-	ok, found := containsElement(s, contains)
+	ok, found := includeElement(s, contains)
 	if !ok {
 		return Fail(t, fmt.Sprintf("%#v could not be applied builtin len()", s), msgAndArgs...)
 	}
@@ -791,7 +787,7 @@ func NotContains(t TestingT, s, contains interface{}, msgAndArgs ...interface{})
 		h.Helper()
 	}
 
-	ok, found := containsElement(s, contains)
+	ok, found := includeElement(s, contains)
 	if !ok {
 		return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", s), msgAndArgs...)
 	}
@@ -835,7 +831,7 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok
 
 	for i := 0; i < subsetValue.Len(); i++ {
 		element := subsetValue.Index(i).Interface()
-		ok, found := containsElement(list, element)
+		ok, found := includeElement(list, element)
 		if !ok {
 			return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", list), msgAndArgs...)
 		}
@@ -856,7 +852,7 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{})
 		h.Helper()
 	}
 	if subset == nil {
-		return Fail(t, "nil is the empty set which is a subset of every set", msgAndArgs...)
+		return Fail(t, fmt.Sprintf("nil is the empty set which is a subset of every set"), msgAndArgs...)
 	}
 
 	subsetValue := reflect.ValueOf(subset)
@@ -879,7 +875,7 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{})
 
 	for i := 0; i < subsetValue.Len(); i++ {
 		element := subsetValue.Index(i).Interface()
-		ok, found := containsElement(list, element)
+		ok, found := includeElement(list, element)
 		if !ok {
 			return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", list), msgAndArgs...)
 		}
@@ -1004,21 +1000,27 @@ func Condition(t TestingT, comp Comparison, msgAndArgs ...interface{}) bool {
 type PanicTestFunc func()
 
 // didPanic returns true if the function passed to it panics. Otherwise, it returns false.
-func didPanic(f PanicTestFunc) (didPanic bool, message interface{}, stack string) {
-	didPanic = true
+func didPanic(f PanicTestFunc) (bool, interface{}, string) {
+
+	didPanic := false
+	var message interface{}
+	var stack string
+	func() {
+
+		defer func() {
+			if message = recover(); message != nil {
+				didPanic = true
+				stack = string(debug.Stack())
+			}
+		}()
+
+		// call the target function
+		f()
 
-	defer func() {
-		message = recover()
-		if didPanic {
-			stack = string(debug.Stack())
-		}
 	}()
 
-	// call the target function
-	f()
-	didPanic = false
+	return didPanic, message, stack
 
-	return
 }
 
 // Panics asserts that the code inside the specified PanicTestFunc panics.
@@ -1159,15 +1161,11 @@ func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs
 	bf, bok := toFloat(actual)
 
 	if !aok || !bok {
-		return Fail(t, "Parameters must be numerical", msgAndArgs...)
-	}
-
-	if math.IsNaN(af) && math.IsNaN(bf) {
-		return true
+		return Fail(t, fmt.Sprintf("Parameters must be numerical"), msgAndArgs...)
 	}
 
 	if math.IsNaN(af) {
-		return Fail(t, "Expected must not be NaN", msgAndArgs...)
+		return Fail(t, fmt.Sprintf("Expected must not be NaN"), msgAndArgs...)
 	}
 
 	if math.IsNaN(bf) {
@@ -1190,7 +1188,7 @@ func InDeltaSlice(t TestingT, expected, actual interface{}, delta float64, msgAn
 	if expected == nil || actual == nil ||
 		reflect.TypeOf(actual).Kind() != reflect.Slice ||
 		reflect.TypeOf(expected).Kind() != reflect.Slice {
-		return Fail(t, "Parameters must be slice", msgAndArgs...)
+		return Fail(t, fmt.Sprintf("Parameters must be slice"), msgAndArgs...)
 	}
 
 	actualSlice := reflect.ValueOf(actual)
@@ -1252,12 +1250,8 @@ func InDeltaMapValues(t TestingT, expected, actual interface{}, delta float64, m
 
 func calcRelativeError(expected, actual interface{}) (float64, error) {
 	af, aok := toFloat(expected)
-	bf, bok := toFloat(actual)
-	if !aok || !bok {
-		return 0, fmt.Errorf("Parameters must be numerical")
-	}
-	if math.IsNaN(af) && math.IsNaN(bf) {
-		return 0, nil
+	if !aok {
+		return 0, fmt.Errorf("expected value %q cannot be converted to float", expected)
 	}
 	if math.IsNaN(af) {
 		return 0, errors.New("expected value must not be NaN")
@@ -1265,6 +1259,10 @@ func calcRelativeError(expected, actual interface{}) (float64, error) {
 	if af == 0 {
 		return 0, fmt.Errorf("expected value must have a value other than zero to calculate the relative error")
 	}
+	bf, bok := toFloat(actual)
+	if !bok {
+		return 0, fmt.Errorf("actual value %q cannot be converted to float", actual)
+	}
 	if math.IsNaN(bf) {
 		return 0, errors.New("actual value must not be NaN")
 	}
@@ -1300,7 +1298,7 @@ func InEpsilonSlice(t TestingT, expected, actual interface{}, epsilon float64, m
 	if expected == nil || actual == nil ||
 		reflect.TypeOf(actual).Kind() != reflect.Slice ||
 		reflect.TypeOf(expected).Kind() != reflect.Slice {
-		return Fail(t, "Parameters must be slice", msgAndArgs...)
+		return Fail(t, fmt.Sprintf("Parameters must be slice"), msgAndArgs...)
 	}
 
 	actualSlice := reflect.ValueOf(actual)
@@ -1377,27 +1375,6 @@ func EqualError(t TestingT, theError error, errString string, msgAndArgs ...inte
 	return true
 }
 
-// ErrorContains asserts that a function returned an error (i.e. not `nil`)
-// and that the error contains the specified substring.
-//
-//   actualObj, err := SomeFunction()
-//   assert.ErrorContains(t, err,  expectedErrorSubString)
-func ErrorContains(t TestingT, theError error, contains string, msgAndArgs ...interface{}) bool {
-	if h, ok := t.(tHelper); ok {
-		h.Helper()
-	}
-	if !Error(t, theError, msgAndArgs...) {
-		return false
-	}
-
-	actual := theError.Error()
-	if !strings.Contains(actual, contains) {
-		return Fail(t, fmt.Sprintf("Error %#v does not contain %#v", actual, contains), msgAndArgs...)
-	}
-
-	return true
-}
-
 // matchRegexp return true if a specified regexp matches a string.
 func matchRegexp(rx interface{}, str interface{}) bool {
 
@@ -1611,17 +1588,12 @@ func diff(expected interface{}, actual interface{}) string {
 	}
 
 	var e, a string
-
-	switch et {
-	case reflect.TypeOf(""):
-		e = reflect.ValueOf(expected).String()
-		a = reflect.ValueOf(actual).String()
-	case reflect.TypeOf(time.Time{}):
-		e = spewConfigStringerEnabled.Sdump(expected)
-		a = spewConfigStringerEnabled.Sdump(actual)
-	default:
+	if et != reflect.TypeOf("") {
 		e = spewConfig.Sdump(expected)
 		a = spewConfig.Sdump(actual)
+	} else {
+		e = reflect.ValueOf(expected).String()
+		a = reflect.ValueOf(actual).String()
 	}
 
 	diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
@@ -1653,14 +1625,6 @@ var spewConfig = spew.ConfigState{
 	MaxDepth:                10,
 }
 
-var spewConfigStringerEnabled = spew.ConfigState{
-	Indent:                  " ",
-	DisablePointerAddresses: true,
-	DisableCapacities:       true,
-	SortKeys:                true,
-	MaxDepth:                10,
-}
-
 type tHelper interface {
 	Helper()
 }