Repository: Azure/ARO-RP
Pull request #1886 (2021-12-21 22:24:05)

Make keyvault client cloud aware
### What this PR does / why we need it:

The current keyvault client used in e2e tests is not cloud aware and is hardcoded to the public cloud ARM endpoint.  This makes it cloud aware like the rest of the clients.  

Note: this branch being merged is at 9b4fecf + this commit, which means that once it is merged into master we can use the non-merge commit as a head for deploying to gov cloud.  

### Test plan for issue:

Run e2e in Public & Gov Cloud INT environments.  

### Is there any documentation that needs to be updated for this PR?

Nope.  


Patch:
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index f351903e3..600e8637e 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1,1 +1,1 @@
-* @jewzaam @m1kola @bennerv @hawkowl @mwoodson @rogbas @petrkotas
+* @jewzaam @m1kola @bennerv @hawkowl @mwoodson @rogbas
diff --git a/.pipelines/onebranch/pipeline.image.mirror.official.yml b/.pipelines/onebranch/pipeline.image.mirror.official.yml
new file mode 100644
index 000000000..5f4231816
--- /dev/null
+++ b/.pipelines/onebranch/pipeline.image.mirror.official.yml
@@ -0,0 +1,51 @@
+#################################################################################
+#                               OneBranch Pipelines                             #
+# This pipeline was created by EasyStart from a sample located at:              #
+#   https://aka.ms/obpipelines/easystart/samples                                #
+# Documentation:  https://aka.ms/obpipelines                                    #
+# Yaml Schema:    https://aka.ms/obpipelines/yaml/schema                        #
+# Retail Tasks:   https://aka.ms/obpipelines/tasks                              #
+# Support:        https://aka.ms/onebranchsup                                   #
+#################################################################################
+
+trigger: none
+pr: none
+
+parameters:
+- name: vsoDeployerBuildID
+  type: string
+  default: latest
+
+variables:
+  Cdp_Definition_Build_Count: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
+  LinuxContainerImage: cdpxlinux.azurecr.io/user/aro/ubi8-gotoolset-1.14.12:2021.10.12   # Docker image which is used to build the project https://aka.ms/obpipelines/containers
+  Debian_Frontend: noninteractive
+
+resources:
+  repositories:
+  - repository: templates
+    type: git
+    name: OneBranch.Pipelines/GovernedTemplates
+    ref: refs/heads/main
+  - repository: rhado
+    type: git
+    name: ARO.Pipelines
+
+extends:
+  template: v2/OneBranch.Official.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates
+  parameters:
+    globalSdl: # https://aka.ms/obpipelines/sdl
+      # tsa:
+      #  enabled: true # SDL results of non-official builds aren't uploaded to TSA by default.
+      # credscan:
+      #   suppressionsFile: $(Build.SourcesDirectory)\.config\CredScanSuppressions.json
+      policheck:
+        break: true # always break the build on policheck issues. You can disable it by setting to 'false'
+      suppression:
+        suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress
+        suppressionSet: default
+
+    stages:
+    - stage: Mirror_Images
+      jobs:
+      - template: .pipelines/onebranch/templates/template-image-mirror.yml@self
diff --git a/.pipelines/onebranch/pipeline.image.mirror.pullrequest.yml b/.pipelines/onebranch/pipeline.image.mirror.pullrequest.yml
new file mode 100644
index 000000000..e35296a6e
--- /dev/null
+++ b/.pipelines/onebranch/pipeline.image.mirror.pullrequest.yml
@@ -0,0 +1,54 @@
+#################################################################################
+#                               OneBranch Pipelines                             #
+# This pipeline was created by EasyStart from a sample located at:              #
+#   https://aka.ms/obpipelines/easystart/samples                                #
+# Documentation:  https://aka.ms/obpipelines                                    #
+# Yaml Schema:    https://aka.ms/obpipelines/yaml/schema                        #
+# Retail Tasks:   https://aka.ms/obpipelines/tasks                              #
+# Support:        https://aka.ms/onebranchsup                                   #
+#################################################################################
+
+trigger: none
+pr: none
+
+parameters:
+- name: vsoDeployerBuildID
+  type: string
+  default: latest
+
+variables:
+  Cdp_Definition_Build_Count: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
+  LinuxContainerImage: cdpxlinux.azurecr.io/user/aro/ubi8-gotoolset-1.14.12:2021.10.12   # Docker image which is used to build the project https://aka.ms/obpipelines/containers
+  Debian_Frontend: noninteractive
+
+resources:
+  repositories:
+  - repository: templates
+    type: git
+    name: OneBranch.Pipelines/GovernedTemplates
+    ref: refs/heads/main
+  - repository: rhado
+    type: git
+    name: ARO.Pipelines
+  - repository: rpconfig
+    type: git
+    name: RP-Config
+
+extends:
+  template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates
+  parameters:
+    globalSdl: # https://aka.ms/obpipelines/sdl
+      # tsa:
+      #  enabled: true # SDL results of non-official builds aren't uploaded to TSA by default.
+      # credscan:
+      #   suppressionsFile: $(Build.SourcesDirectory)\.config\CredScanSuppressions.json
+      policheck:
+        break: true # always break the build on policheck issues. You can disable it by setting to 'false'
+      suppression:
+        suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress
+        suppressionSet: default
+
+    stages:
+    - stage: Mirror_Images
+      jobs:
+      - template: .pipelines/onebranch/templates/template-image-mirror.yml@self
diff --git a/.pipelines/onebranch/scripts/mirroring.sh b/.pipelines/onebranch/scripts/mirroring.sh
new file mode 100644
index 000000000..106d5c197
--- /dev/null
+++ b/.pipelines/onebranch/scripts/mirroring.sh
@@ -0,0 +1,13 @@
+SERVICE_GROUP_ROOT=$BUILD_SOURCESDIRECTORY/ARO.Pipelines/ev2/Mirroring/ServiceGroupRoot
+EV2_BIN=$SERVICE_GROUP_ROOT/bin
+ARO_DIR=$BUILD_SOURCESDIRECTORY/deployer
+
+mkdir $OB_OUTPUTDIRECTORY
+
+cd $ARO_DIR
+tar -cvf $EV2_BIN/aro.tar aro
+cd $SERVICE_GROUP_ROOT/bin
+tar -rf $EV2_BIN/aro.tar mirror.sh
+rm mirror.sh
+
+cp -r $SERVICE_GROUP_ROOT $OB_OUTPUTDIRECTORY/ServiceGroupRoot/
diff --git a/.pipelines/onebranch/templates/template-image-mirror.yml b/.pipelines/onebranch/templates/template-image-mirror.yml
new file mode 100644
index 000000000..4f5776012
--- /dev/null
+++ b/.pipelines/onebranch/templates/template-image-mirror.yml
@@ -0,0 +1,28 @@
+jobs:
+- job: Mirror_Images
+  timeoutInMinutes: 360
+  pool:
+    type: linux
+
+  variables:   # More settings at https://aka.ms/obpipelines/yaml/jobs
+    ob_outputDirectory: $(Build.SourcesDirectory)/out     # this directory is uploaded to pipeline artifacts, reddog and cloudvault. More info at https://aka.ms/obpipelines/artifacts
+    vso-deployer-pipeline-id: 206753
+    vso-project-id: 5d69ab04-7ded-49dc-84d5-bbbcac4add8d
+
+  steps:
+  - checkout: rhado
+  - task: DownloadPipelineArtifact@2
+    displayName: Download Deployer
+    inputs:
+      buildType: specific
+      project: $(vso-project-id)
+      pipeline: $(vso-deployer-pipeline-id)
+      ${{ if eq(parameters.vsoDeployerBuildID, 'latest') }}:
+        buildVersionToDownload: FromBranch
+        branchName: refs/heads/master
+      downloadType: specific
+      downloadPath: $(Build.SourcesDirectory)/deployer
+      artifact: drop_Build_ARO_Build_ARO
+  - template: ./template-generate-ev2-manifests.yml
+    parameters:
+      generationType: mirroring
diff --git a/cmd/aro/mirror.go b/cmd/aro/mirror.go
index 559c3d6a7..d7fabc688 100644
--- a/cmd/aro/mirror.go
+++ b/cmd/aro/mirror.go
@@ -87,7 +87,7 @@ func mirror(ctx context.Context, log *logrus.Entry) error {
 	var releases []pkgmirror.Node
 	if len(flag.Args()) == 1 {
 		log.Print("reading release graph")
-		releases, err = pkgmirror.AddFromGraph(version.NewVersion(4, 6))
+		releases, err = pkgmirror.AddFromGraph(version.NewVersion(4, 3))
 		if err != nil {
 			return err
 		}
diff --git a/docs/deploy-full-rp-service-in-dev.md b/docs/deploy-full-rp-service-in-dev.md
index a2746d0f2..b07b7b93e 100644
--- a/docs/deploy-full-rp-service-in-dev.md
+++ b/docs/deploy-full-rp-service-in-dev.md
@@ -101,7 +101,7 @@
     1. Setup mirroring environment variables
         ```bash
         export DST_ACR_NAME=${USER}aro
-        export SRC_AUTH_QUAY=$(echo $USER_PULL_SECRET | jq -r '.auths."quay.io".auth')
+        export SRC_AUTH_QUAY=FILL_IN # Get quay auth https://cloud.redhat.com/openshift/create/local -> Download Pull Secret. Use base64 value from quay.io part
         export SRC_AUTH_REDHAT=$(echo $USER_PULL_SECRET | jq -r '.auths."registry.redhat.io".auth')
         export DST_AUTH=$(echo -n '00000000-0000-0000-0000-000000000000:'$(az acr login -n ${DST_ACR_NAME} --expose-token | jq -r .accessToken) | base64 -w0)
 
@@ -116,10 +116,9 @@
         go run -tags aro ./cmd/aro mirror latest
         ```
 
-    1. Push the ARO and Fluentbit images to your ACR
+    1. Push the ARO image to your ACR
         ```bash
         make publish-image-aro-multistage
-        make publish-image-fluentbit
         ```
 
 1. Update the DNS Child Domains
diff --git a/go.mod b/go.mod
index f004a8477..4fdaeae7a 100644
--- a/go.mod
+++ b/go.mod
@@ -140,13 +140,6 @@ require (
 )
 
 exclude (
-	// exclude github.com/golang/protobuf < 1.3.2 https://nvd.nist.gov/vuln/detail/CVE-2021-3121
-	github.com/golang/protobuf v1.0.0
-	github.com/golang/protobuf v1.1.1
-	github.com/golang/protobuf v1.2.0
-	github.com/golang/protobuf v1.2.1
-	github.com/golang/protobuf v1.3.0
-	github.com/golang/protobuf v1.3.1
 	// exclude github.com/hashicorp/vault < v1.5.1: https://nvd.nist.gov/vuln/detail/CVE-2020-16251
 	github.com/hashicorp/vault v0.10.4
 	github.com/hashicorp/vault v0.11.0
@@ -211,8 +204,6 @@ replace (
 	bitbucket.org/ww/goautoneg => github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d // 404 on bitbucket.org/ww/goautoneg
 	github.com/docker/spdystream => github.com/docker/spdystream v0.1.0
 	github.com/go-openapi/spec => github.com/go-openapi/spec v0.19.8
-	// Replace old GoGo Protobuf versions https://nvd.nist.gov/vuln/detail/CVE-2021-3121
-	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
 	github.com/operator-framework/operator-sdk => github.com/operator-framework/operator-sdk v0.19.4
 	// https://www.whitesourcesoftware.com/vulnerability-database/WS-2018-0594
 	github.com/satori/go.uuid => github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b
@@ -250,6 +241,7 @@ replace (
 	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.21.4
 	sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.9.1
 	sigs.k8s.io/controller-tools => sigs.k8s.io/controller-tools v0.5.0
+
 )
 
 // Installer dependencies. Some of them are being used directly in the RP.
diff --git a/go.sum b/go.sum
index 83bb6a4b0..ffcbbdc9d 100644
--- a/go.sum
+++ b/go.sum
@@ -979,6 +979,13 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
 github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
+github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A=
@@ -1012,6 +1019,8 @@ github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3K
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
@@ -1563,6 +1572,7 @@ github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT
 github.com/keybase/go-crypto v0.0.0-20161004153544-93f5b35093ba/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M=
 github.com/keybase/go-crypto v0.0.0-20190403132359-d65b6b94177f/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M=
 github.com/keybase/go-crypto v0.0.0-20200123153347-de78d2cb44f4/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
diff --git a/pkg/deploy/devconfig.go b/pkg/deploy/devconfig.go
index 2fcd5ef46..cc8266bd6 100644
--- a/pkg/deploy/devconfig.go
+++ b/pkg/deploy/devconfig.go
@@ -137,7 +137,7 @@ func DevConfig(_env env.Core) (*Config, error) {
 				adminKeyvaultAccessPolicy(_env),
 				deployKeyvaultAccessPolicy(_env),
 			},
-			FluentbitImage:       to.StringPtr(version.FluentbitImage(os.Getenv("USER") + _env.Environment().ContainerRegistryDNSSuffix)),
+			FluentbitImage:       to.StringPtr(os.Getenv("FLUENTBIT_IMAGE")),
 			FPClientID:           to.StringPtr(os.Getenv("AZURE_FP_CLIENT_ID")),
 			FPServicePrincipalID: to.StringPtr(os.Getenv("AZURE_FP_SERVICE_PRINCIPAL_ID")),
 			GatewayDomains: []string{
@@ -192,7 +192,6 @@ func DevConfig(_env env.Core) (*Config, error) {
 			SSHPublicKey:                      to.StringPtr(string(sshPublicKey)),
 			SubscriptionResourceGroupLocation: to.StringPtr(_env.Location()),
 			SubscriptionResourceGroupName:     to.StringPtr(os.Getenv("USER") + "-subscription"),
-			VMSSCleanupEnabled:                to.BoolPtr(true),
 			VMSize:                            to.StringPtr("Standard_D2s_v3"),
 		},
 	}, nil
diff --git a/pkg/util/azureclient/mgmt/keyvault/vaults.go b/pkg/util/azureclient/mgmt/keyvault/vaults.go
index f7b9b9e7b..2d2ebf52c 100644
--- a/pkg/util/azureclient/mgmt/keyvault/vaults.go
+++ b/pkg/util/azureclient/mgmt/keyvault/vaults.go
@@ -8,6 +8,8 @@ import (
 
 	mgmtkeyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2019-09-01/keyvault"
 	"github.com/Azure/go-autorest/autorest"
+
+	"github.com/Azure/ARO-RP/pkg/util/azureclient"
 )
 
 // VaultsClient is a minimal interface for azure VaultsClient
@@ -21,9 +23,9 @@ type vaultsClient struct {
 
 var _ VaultsClient = &vaultsClient{}
 
-// NewDisksClient creates a new DisksClient
-func NewVaultsClient(subscriptionID string, authorizer autorest.Authorizer) VaultsClient {
-	client := mgmtkeyvault.NewVaultsClient(subscriptionID)
+// NewVaultsClient creates a new KeyvaultClient
+func NewVaultsClient(environment *azureclient.AROEnvironment, subscriptionID string, authorizer autorest.Authorizer) VaultsClient {
+	client := mgmtkeyvault.NewVaultsClientWithBaseURI(environment.ResourceManagerEndpoint, subscriptionID)
 	client.Authorizer = authorizer
 
 	return &vaultsClient{
diff --git a/pkg/util/clientauthorizer/arm.go b/pkg/util/clientauthorizer/arm.go
index 8db940b35..ab09cda47 100644
--- a/pkg/util/clientauthorizer/arm.go
+++ b/pkg/util/clientauthorizer/arm.go
@@ -9,12 +9,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"reflect"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/sirupsen/logrus"
 
 	"github.com/Azure/ARO-RP/pkg/util/instancemetadata"
@@ -103,13 +101,7 @@ func (a *arm) refresh() {
 func (a *arm) refreshOnce() error {
 	now := a.now()
 
-	// ARM <-> RP Authentication endpoint is not consistent.  Check ARM wiki for up-to-date metadata endpoints
-	endpoint := strings.TrimSuffix(a.im.Environment().ResourceManagerEndpoint, "/") + ":24582"
-	if reflect.DeepEqual(a.im.Environment().Environment, azure.PublicCloud) {
-		endpoint = "https://admin.management.azure.com"
-	}
-
-	req, err := http.NewRequest(http.MethodGet, endpoint+"/metadata/authentication?api-version=2015-01-01", nil)
+	req, err := http.NewRequest(http.MethodGet, strings.TrimSuffix(a.im.Environment().ResourceManagerEndpoint, "/")+":24582/metadata/authentication?api-version=2015-01-01", nil)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/util/cluster/cluster.go b/pkg/util/cluster/cluster.go
index 366191695..fc8903d45 100644
--- a/pkg/util/cluster/cluster.go
+++ b/pkg/util/cluster/cluster.go
@@ -112,7 +112,7 @@ func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) {
 		routetables:                       network.NewRouteTablesClient(environment.Environment(), environment.SubscriptionID(), authorizer),
 		roleassignments:                   authorization.NewRoleAssignmentsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
 		peerings:                          network.NewVirtualNetworkPeeringsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
-		vaultsClient:                      keyvaultclient.NewVaultsClient(environment.SubscriptionID(), authorizer),
+		vaultsClient:                      keyvaultclient.NewVaultsClient(environment.Environment(), environment.SubscriptionID(), authorizer),
 	}
 
 	return c, nil
diff --git a/vendor/golang.org/x/oauth2/microsoft/microsoft.go b/vendor/golang.org/x/oauth2/microsoft/microsoft.go
new file mode 100644
index 000000000..3ffbc57a6
--- /dev/null
+++ b/vendor/golang.org/x/oauth2/microsoft/microsoft.go
@@ -0,0 +1,31 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package microsoft provides constants for using OAuth2 to access Windows Live ID.
+package microsoft // import "golang.org/x/oauth2/microsoft"
+
+import (
+	"golang.org/x/oauth2"
+)
+
+// LiveConnectEndpoint is Windows's Live ID OAuth 2.0 endpoint.
+var LiveConnectEndpoint = oauth2.Endpoint{
+	AuthURL:  "https://login.live.com/oauth20_authorize.srf",
+	TokenURL: "https://login.live.com/oauth20_token.srf",
+}
+
+// AzureADEndpoint returns a new oauth2.Endpoint for the given tenant at Azure Active Directory.
+// If tenant is empty, it uses the tenant called `common`.
+//
+// For more information see:
+// https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-v2-protocols#endpoints
+func AzureADEndpoint(tenant string) oauth2.Endpoint {
+	if tenant == "" {
+		tenant = "common"
+	}
+	return oauth2.Endpoint{
+		AuthURL:  "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/authorize",
+		TokenURL: "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/token",
+	}
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 76c2d9d7a..d8db4d05f 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -393,7 +393,7 @@ github.com/gofrs/flock
 # github.com/gofrs/uuid v4.0.0+incompatible
 ## explicit
 github.com/gofrs/uuid
-# github.com/gogo/protobuf v1.3.2 => github.com/gogo/protobuf v1.3.2
+# github.com/gogo/protobuf v1.3.2
 github.com/gogo/protobuf/proto
 github.com/gogo/protobuf/sortkeys
 # github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
@@ -1217,6 +1217,7 @@ golang.org/x/oauth2/google/internal/externalaccount
 golang.org/x/oauth2/internal
 golang.org/x/oauth2/jws
 golang.org/x/oauth2/jwt
+golang.org/x/oauth2/microsoft
 # golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 ## explicit
 golang.org/x/sync/errgroup
@@ -2038,7 +2039,6 @@ sigs.k8s.io/yaml
 # bitbucket.org/ww/goautoneg => github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d
 # github.com/docker/spdystream => github.com/docker/spdystream v0.1.0
 # github.com/go-openapi/spec => github.com/go-openapi/spec v0.19.8
-# github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
 # github.com/operator-framework/operator-sdk => github.com/operator-framework/operator-sdk v0.19.4
 # github.com/satori/go.uuid => github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b
 # github.com/satori/uuid => github.com/satori/uuid v1.2.1-0.20181028125025-b2ce2384e17b


Test patch:
diff --git a/.pipelines/e2e.yml b/.pipelines/e2e.yml
index 21b26b434..63130c655 100644
--- a/.pipelines/e2e.yml
+++ b/.pipelines/e2e.yml
@@ -21,7 +21,6 @@ jobs:
     parameters:
       azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn)
   - script: |
-      az account set -s $AZURE_SUBSCRIPTION_ID
       export SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME)
       make secrets
       . secrets/env
diff --git a/pkg/util/clientauthorizer/arm_test.go b/pkg/util/clientauthorizer/arm_test.go
index 0dbd43ecc..7045cd9db 100644
--- a/pkg/util/clientauthorizer/arm_test.go
+++ b/pkg/util/clientauthorizer/arm_test.go
@@ -10,12 +10,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"reflect"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/golang/mock/gomock"
 
 	"github.com/Azure/ARO-RP/pkg/util/azureclient"
@@ -24,14 +22,12 @@ import (
 
 func TestARMRefreshOnce(t *testing.T) {
 	for _, tt := range []struct {
-		name     string
-		azureEnv azureclient.AROEnvironment
-		do       func(*http.Request) (*http.Response, error)
-		wantErr  string
+		name    string
+		do      func(*http.Request) (*http.Response, error)
+		wantErr string
 	}{
 		{
-			name:     "valid public cloud",
-			azureEnv: azureclient.PublicCloud,
+			name: "valid",
 			do: func(*http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusOK,
@@ -52,30 +48,7 @@ func TestARMRefreshOnce(t *testing.T) {
 			},
 		},
 		{
-			name:     "valid gov cloud",
-			azureEnv: azureclient.USGovernmentCloud,
-			do: func(*http.Request) (*http.Response, error) {
-				return &http.Response{
-					StatusCode: http.StatusOK,
-					Header: http.Header{
-						"Content-Type": []string{"application/json; charset=utf-8"},
-					},
-					Body: ioutil.NopCloser(strings.NewReader(
-						`{
-							"clientCertificates": [
-								{
-									"notBefore": "2020-01-19T23:00:00Z",
-									"notAfter": "2020-01-20T01:00:00Z"
-								}
-							]
-						}`,
-					)),
-				}, nil
-			},
-		},
-		{
-			name:     "invalid - no certificate for now",
-			azureEnv: azureclient.PublicCloud,
+			name: "invalid - no certificate for now",
 			do: func(*http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusOK,
@@ -97,8 +70,7 @@ func TestARMRefreshOnce(t *testing.T) {
 			wantErr: "did not receive current certificate",
 		},
 		{
-			name:     "invalid JSON",
-			azureEnv: azureclient.PublicCloud,
+			name: "invalid JSON",
 			do: func(*http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusOK,
@@ -111,16 +83,14 @@ func TestARMRefreshOnce(t *testing.T) {
 			wantErr: "invalid character 'o' in literal null (expecting 'u')",
 		},
 		{
-			name:     "invalid - error",
-			azureEnv: azureclient.PublicCloud,
+			name: "invalid - error",
 			do: func(*http.Request) (*http.Response, error) {
 				return nil, errors.New("fake error")
 			},
 			wantErr: "fake error",
 		},
 		{
-			name:     "invalid - status code",
-			azureEnv: azureclient.PublicCloud,
+			name: "invalid - status code",
 			do: func(*http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusBadGateway,
@@ -130,8 +100,7 @@ func TestARMRefreshOnce(t *testing.T) {
 			wantErr: "unexpected status code 502",
 		},
 		{
-			name:     "invalid - content type",
-			azureEnv: azureclient.PublicCloud,
+			name: "invalid - content type",
 			do: func(*http.Request) (*http.Response, error) {
 				return &http.Response{
 					StatusCode: http.StatusOK,
@@ -149,7 +118,7 @@ func TestARMRefreshOnce(t *testing.T) {
 			defer controller.Finish()
 
 			im := mock_instancemetadata.NewMockInstanceMetadata(controller)
-			im.EXPECT().Environment().AnyTimes().Return(&tt.azureEnv)
+			im.EXPECT().Environment().AnyTimes().Return(&azureclient.PublicCloud)
 
 			a := &arm{
 				now: func() time.Time { return time.Date(2020, 1, 20, 0, 0, 0, 0, time.UTC) },
@@ -157,11 +126,7 @@ func TestARMRefreshOnce(t *testing.T) {
 					if req.Method != http.MethodGet {
 						return nil, fmt.Errorf("unexpected method %q", req.Method)
 					}
-					endpoint := strings.TrimSuffix(im.Environment().ResourceManagerEndpoint, "/") + ":24582"
-					if reflect.DeepEqual(im.Environment().Environment, azure.PublicCloud) {
-						endpoint = "https://admin.management.azure.com"
-					}
-					if req.URL.String() != endpoint+"/metadata/authentication?api-version=2015-01-01" {
+					if req.URL.String() != strings.TrimSuffix(im.Environment().ResourceManagerEndpoint, "/")+":24582/metadata/authentication?api-version=2015-01-01" {
 						return nil, fmt.Errorf("unexpected URL %q", req.URL.String())
 					}
 					return tt.do(req)
diff --git a/test/e2e/operator.go b/test/e2e/operator.go
index 93bfa8c08..a77a81372 100644
--- a/test/e2e/operator.go
+++ b/test/e2e/operator.go
@@ -278,10 +278,14 @@ var _ = Describe("ARO Operator - Conditions", func() {
 	})
 })
 
-var _ = Describe("ARO Operator - MachineSet Controller", func() {
+var _ = XDescribe("ARO Operator - MachineSet Controller", func() {
 	Specify("operator should maintain at least two worker replicas", func() {
 		ctx := context.Background()
 
+		// TODO: MSFT Billing expects that we only scale a single node (4 VMs).
+		// Need to work with billing pipeline to ensure we can run operator tests
+		skipIfNotInDevelopmentEnv()
+
 		instance, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{})
 		Expect(err).NotTo(HaveOccurred())
 
@@ -293,13 +297,11 @@ var _ = Describe("ARO Operator - MachineSet Controller", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(mss.Items).NotTo(BeEmpty())
 
-		// Zero all machinesets, wait for reconcile
+		// Zero all machinesets (avoid availability zone confusion)
 		for _, object := range mss.Items {
 			err = scale(object.Name, 0)
 			Expect(err).NotTo(HaveOccurred())
-		}
 
-		for _, object := range mss.Items {
 			err = waitForScale(object.Name)
 			Expect(err).NotTo(HaveOccurred())
 		}
@@ -316,20 +318,11 @@ var _ = Describe("ARO Operator - MachineSet Controller", func() {
 		}
 		Expect(replicaCount).To(BeEquivalentTo(minSupportedReplicas))
 
-		// Scale back to previous state
+		// Restore previous state
 		for _, ms := range mss.Items {
-			err = scale(ms.Name, *ms.Spec.Replicas)
+			err := scale(ms.Name, *ms.Spec.Replicas)
 			Expect(err).NotTo(HaveOccurred())
 		}
-
-		for _, ms := range mss.Items {
-			err = waitForScale(ms.Name)
-			Expect(err).NotTo(HaveOccurred())
-		}
-
-		// Wait for old machine objects to delete
-		err = waitForMachines()
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
 
diff --git a/test/e2e/scalenodes.go b/test/e2e/scalenodes.go
index 49c97fa0e..c47c31c9c 100644
--- a/test/e2e/scalenodes.go
+++ b/test/e2e/scalenodes.go
@@ -73,13 +73,6 @@ var _ = Describe("Scale nodes", func() {
 	Specify("nodes should scale up and down", func() {
 		ctx := context.Background()
 
-		instance, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, "cluster", metav1.GetOptions{})
-		Expect(err).NotTo(HaveOccurred())
-
-		if instance.Spec.Features.ReconcileMachineSet {
-			Skip("MachineSet Controller is enabled, skipping this test")
-		}
-
 		mss, err := clients.MachineAPI.MachineV1beta1().MachineSets(machineSetsNamespace).List(ctx, metav1.ListOptions{})
 		Expect(err).NotTo(HaveOccurred())
 		Expect(mss.Items).NotTo(BeEmpty())
@@ -119,36 +112,17 @@ func scale(name string, replicas int32) error {
 
 func waitForScale(name string) error {
 	return wait.PollImmediate(10*time.Second, 30*time.Minute, func() (bool, error) {
-		machineset, err := clients.MachineAPI.MachineV1beta1().MachineSets(machineSetsNamespace).Get(context.Background(), name, metav1.GetOptions{})
+		ms, err := clients.MachineAPI.MachineV1beta1().MachineSets(machineSetsNamespace).Get(context.Background(), name, metav1.GetOptions{})
 		if err != nil {
 			log.Warn(err)
 			return false, nil // swallow error
 		}
 
-		if machineset.Spec.Replicas == nil {
-			return false, nil
-		}
-
-		return machineset.Status.ObservedGeneration == machineset.Generation &&
-			machineset.Status.AvailableReplicas == *machineset.Spec.Replicas, nil
-	})
-}
-
-func waitForMachines() error {
-	return wait.PollImmediate(1*time.Second, 30*time.Minute, func() (bool, error) {
-		machines, err := clients.MachineAPI.MachineV1beta1().Machines(machineSetsNamespace).List(context.Background(), metav1.ListOptions{})
-		if err != nil {
-			log.Warn(err)
+		if ms.Spec.Replicas == nil {
 			return false, nil
 		}
 
-		// Wait for all machines to be in Running phase before continuing
-		for _, m := range machines.Items {
-			if *m.Status.Phase != "Running" {
-				return false, nil
-			}
-		}
-
-		return true, nil
+		return ms.Status.ObservedGeneration == ms.Generation &&
+			ms.Status.AvailableReplicas == *ms.Spec.Replicas, nil
 	})
 }