Repository: Azure/ARO-RP
Pull request #1131 (2020-11-09 13:31:45)

Discovery cache hack 
### Which issue this PR addresses:

Fixes "Internal Server Error" when live and cache discovery fails. 

### What this PR does / why we need it:

In cases where we have a dead `APIService` object, we get internal server error. This hack ignores the error

### Test plan for issue:

`oc new-project test`
```
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
  name: v1beta1.test.faros.sh
spec:
  caBundle: fakebundle
  group: test.faros.sh
  groupPriorityMinimum: 100
  service:
    name: test
    namespace: test
    port: 443
  version: v1beta1
  versionPriority: 100
```
`of create -f api.yaml`

```
# test
curl -k -X GET      -H 'Content-Type: application/json'      "https://localhost:8443/admin/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourcegroups/${RESOURCEGROUP}/providers/Microsoft.RedHatOpenShift/openShiftClusters/${CLUSTER}/kubernetesobjects?api-version=admin&kind=test.example.com"
curl -k -X GET      -H 'Content-Type: application/json'      "https://localhost:8443/admin/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourcegroups/${RESOURCEGROUP}/providers/Microsoft.RedHatOpenShift/openShiftClusters/${CLUSTER}/kubernetesobjects?api-version=admin&kind=ClusterVersion" 

```

### Is there any documentation that needs to be updated for this PR?

N/A


jim-minter: OK, I think the following is a better way: in `RefreshAPIResources()`, just do something like:

```golang
	_, dh.apiresources, err = cli.ServerGroupsAndResources()
	if discovery.IsGroupDiscoveryFailedError(err) {
		// Some group discovery failed; dh.apiresources will have all the ones
		// that worked.  This error can happen with a misconfigured apiservice,
		// for example.  Log it and try to keep going.
		dh.log.Warn(err)
		return nil
	}

	return err
```

Patch:
diff --git a/.pipelines/int-release.yml b/.pipelines/int-release.yml
index 48bca103f..6b62f1f46 100644
--- a/.pipelines/int-release.yml
+++ b/.pipelines/int-release.yml
@@ -4,11 +4,6 @@ trigger: none
 
 parameters:
 - name: vsoDeployerBuildID
-  type: string
-  default: master
-- name: vsoConfigBuildID
-  type: string
-  default: master
 - name: fullDeploy
   type: boolean
   default: false
@@ -27,7 +22,6 @@ stages:
       azureDevOpsJSONSPN: $(aro-v4-ci-devops-spn)
       vsoProjectID: $(vso-project-id)
       vsoConfigPipelineID: $(vso-config-pipeline-id)
-      vsoConfigBuildID: ${{ parameters.vsoConfigBuildID }}
       vsoDeployerPipelineID: $(vso-deployer-pipeline-id)
       vsoDeployerBuildID: ${{ parameters.vsoDeployerBuildID }}
       azureDevOpsE2EJSONSPN: $(aro-v4-e2e-devops-spn)
diff --git a/.pipelines/templates/template-job-deploy-azure-env.yml b/.pipelines/templates/template-job-deploy-azure-env.yml
index 5e08681dd..b8d25f46b 100644
--- a/.pipelines/templates/template-job-deploy-azure-env.yml
+++ b/.pipelines/templates/template-job-deploy-azure-env.yml
@@ -32,10 +32,10 @@ jobs:
               buildType: specific
               project: ${{ parameters.vsoProjectID }}
               pipeline: ${{ parameters.vsoConfigPipelineID }}
-              ${{ if eq(parameters.vsoConfigBuildID, 'master') }}:
+              ${{ if eq(parameters.vsoConfigBuildID, '') }}:
                 buildVersionToDownload: latestFromBranch
                 branchName: refs/heads/master
-              ${{ if ne(parameters.vsoConfigBuildID, 'master') }}:
+              ${{ if ne(parameters.vsoConfigBuildID, '') }}:
                 buildVersionToDownload: specific
                 buildId: ${{ parameters.vsoConfigBuildID }}
               downloadType: specific
@@ -46,10 +46,10 @@ jobs:
               buildType: specific
               project: ${{ parameters.vsoProjectID }}
               pipeline: ${{ parameters.vsoDeployerPipelineID }}
-              ${{ if eq(parameters.vsoDeployerBuildID, 'master') }}:
+              ${{ if eq(parameters.vsoDeployerBuildID, '') }}:
                 buildVersionToDownload: latestFromBranch
                 branchName: refs/heads/master
-              ${{ if ne(parameters.vsoDeployerBuildID, 'master') }}:
+              ${{ if ne(parameters.vsoDeployerBuildID, '') }}:
                 buildVersionToDownload: specific
                 buildId: ${{ parameters.vsoDeployerBuildID }}
               downloadType: specific
diff --git a/pkg/backend/backend.go b/pkg/backend/backend.go
index 1f1186f04..77c3a5f8f 100644
--- a/pkg/backend/backend.go
+++ b/pkg/backend/backend.go
@@ -53,22 +53,6 @@ type Runnable interface {
 
 // NewBackend returns a new runnable backend
 func NewBackend(ctx context.Context, log *logrus.Entry, env env.Interface, dbAsyncOperations database.AsyncOperations, dbBilling database.Billing, dbOpenShiftClusters database.OpenShiftClusters, dbSubscriptions database.Subscriptions, cipher encryption.Cipher, m metrics.Interface) (Runnable, error) {
-	b, err := newBackend(ctx, log, env, dbAsyncOperations, dbBilling, dbOpenShiftClusters, dbSubscriptions, cipher, m)
-	if err != nil {
-		return nil, err
-	}
-
-	b.ocb = newOpenShiftClusterBackend(b)
-	b.sb = newSubscriptionBackend(b)
-	return b, nil
-}
-
-func newBackend(ctx context.Context, log *logrus.Entry, env env.Interface, dbAsyncOperations database.AsyncOperations, dbBilling database.Billing, dbOpenShiftClusters database.OpenShiftClusters, dbSubscriptions database.Subscriptions, cipher encryption.Cipher, m metrics.Interface) (*backend, error) {
-	billing, err := billing.NewManager(env, dbBilling, dbSubscriptions, log)
-	if err != nil {
-		return nil, err
-	}
-
 	b := &backend{
 		baseLog: log,
 		env:     env,
@@ -78,12 +62,22 @@ func newBackend(ctx context.Context, log *logrus.Entry, env env.Interface, dbAsy
 		dbOpenShiftClusters: dbOpenShiftClusters,
 		dbSubscriptions:     dbSubscriptions,
 
-		billing: billing,
-		cipher:  cipher,
-		m:       m,
+		cipher: cipher,
+		m:      m,
 	}
+
 	b.cond = sync.NewCond(&b.mu)
 	b.stopping.Store(false)
+
+	b.ocb = &openShiftClusterBackend{backend: b}
+	b.sb = &subscriptionBackend{backend: b}
+
+	var err error
+	b.billing, err = billing.NewManager(env, dbBilling, dbSubscriptions, log)
+	if err != nil {
+		return nil, err
+	}
+
 	return b, nil
 }
 
@@ -130,15 +124,12 @@ func (b *backend) Run(ctx context.Context, stop <-chan struct{}, done chan<- str
 		}
 	}
 
-	b.waitForWorkerCompletion()
-	b.baseLog.Print("exiting")
-	close(done)
-}
-
-func (b *backend) waitForWorkerCompletion() {
 	b.mu.Lock()
 	for atomic.LoadInt32(&b.workers) > 0 {
 		b.cond.Wait()
 	}
 	b.mu.Unlock()
+
+	b.baseLog.Print("exiting")
+	close(done)
 }
diff --git a/pkg/backend/openshiftcluster.go b/pkg/backend/openshiftcluster.go
index 6d43005c0..647202d07 100644
--- a/pkg/backend/openshiftcluster.go
+++ b/pkg/backend/openshiftcluster.go
@@ -16,25 +16,12 @@ import (
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/backend/openshiftcluster"
-	"github.com/Azure/ARO-RP/pkg/database"
-	"github.com/Azure/ARO-RP/pkg/env"
-	"github.com/Azure/ARO-RP/pkg/util/billing"
-	"github.com/Azure/ARO-RP/pkg/util/encryption"
 	utillog "github.com/Azure/ARO-RP/pkg/util/log"
 	"github.com/Azure/ARO-RP/pkg/util/recover"
 )
 
 type openShiftClusterBackend struct {
 	*backend
-
-	newManager func(log *logrus.Entry, _env env.Interface, db database.OpenShiftClusters, cipher encryption.Cipher, billing billing.Manager, doc *api.OpenShiftClusterDocument, subscriptionDoc *api.SubscriptionDocument) (openshiftcluster.Manager, error)
-}
-
-func newOpenShiftClusterBackend(b *backend) *openShiftClusterBackend {
-	return &openShiftClusterBackend{
-		backend:    b,
-		newManager: openshiftcluster.NewManager,
-	}
 }
 
 // try tries to dequeue an OpenShiftClusterDocument for work, and works it on a
@@ -100,7 +87,7 @@ func (ocb *openShiftClusterBackend) handle(ctx context.Context, log *logrus.Entr
 		return err
 	}
 
-	m, err := ocb.newManager(log, ocb.env, ocb.dbOpenShiftClusters, ocb.cipher, ocb.billing, doc, subscriptionDoc)
+	m, err := openshiftcluster.NewManager(log, ocb.env, ocb.dbOpenShiftClusters, ocb.cipher, ocb.billing, doc, subscriptionDoc)
 	if err != nil {
 		return ocb.endLease(ctx, log, stop, doc, api.ProvisioningStateFailed, err)
 	}
diff --git a/pkg/backend/openshiftcluster/adminupdate.go b/pkg/backend/openshiftcluster/adminupdate.go
index 5cd9ea0cb..d0cfd09b9 100644
--- a/pkg/backend/openshiftcluster/adminupdate.go
+++ b/pkg/backend/openshiftcluster/adminupdate.go
@@ -9,7 +9,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/cluster"
 )
 
-func (m *manager) AdminUpdate(ctx context.Context) error {
+func (m *Manager) AdminUpdate(ctx context.Context) error {
 	// m.ocDynamicValidator.Dynamic is not called so that it doesn't block an
 	// admin update
 
diff --git a/pkg/backend/openshiftcluster/create.go b/pkg/backend/openshiftcluster/create.go
index 01b4fb1d9..3d1869732 100644
--- a/pkg/backend/openshiftcluster/create.go
+++ b/pkg/backend/openshiftcluster/create.go
@@ -41,7 +41,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
 
-func (m *manager) Create(ctx context.Context) error {
+func (m *Manager) Create(ctx context.Context) error {
 	var err error
 
 	if m.doc.OpenShiftCluster.Properties.Install == nil {
diff --git a/pkg/backend/openshiftcluster/delete.go b/pkg/backend/openshiftcluster/delete.go
index f67a60bff..8fcded99d 100644
--- a/pkg/backend/openshiftcluster/delete.go
+++ b/pkg/backend/openshiftcluster/delete.go
@@ -9,7 +9,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/cluster"
 )
 
-func (m *manager) Delete(ctx context.Context) error {
+func (m *Manager) Delete(ctx context.Context) error {
 	i, err := cluster.New(ctx, m.log, m.env, m.db, m.cipher, m.billing, m.doc, m.subscriptionDoc)
 	if err != nil {
 		return err
diff --git a/pkg/backend/openshiftcluster/generate.go b/pkg/backend/openshiftcluster/generate.go
deleted file mode 100644
index eb9270614..000000000
--- a/pkg/backend/openshiftcluster/generate.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package openshiftcluster
-
-// Copyright (c) Microsoft Corporation.
-// Licensed under the Apache License 2.0.
-
-//go:generate go run ../../../vendor/github.com/golang/mock/mockgen -destination=../../util/mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/backend/$GOPACKAGE Manager
-//go:generate go run ../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../util/mocks/$GOPACKAGE/$GOPACKAGE.go
diff --git a/pkg/backend/openshiftcluster/openshiftcluster.go b/pkg/backend/openshiftcluster/openshiftcluster.go
index 90a4bad70..3f1653e1a 100644
--- a/pkg/backend/openshiftcluster/openshiftcluster.go
+++ b/pkg/backend/openshiftcluster/openshiftcluster.go
@@ -4,8 +4,6 @@ package openshiftcluster
 // Licensed under the Apache License 2.0.
 
 import (
-	"context"
-
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/sirupsen/logrus"
@@ -26,16 +24,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
 
-type Manager interface {
-	Create(ctx context.Context) error
-	Update(ctx context.Context) error
-	AdminUpdate(ctx context.Context) error
-	Delete(ctx context.Context) error
-}
-
-var _ Manager = &manager{}
-
-type manager struct {
+type Manager struct {
 	log          *logrus.Entry
 	env          env.Interface
 	db           database.OpenShiftClusters
@@ -59,7 +48,7 @@ type manager struct {
 }
 
 // NewManager returns a new openshiftcluster Manager
-func NewManager(log *logrus.Entry, _env env.Interface, db database.OpenShiftClusters, cipher encryption.Cipher, billing billing.Manager, doc *api.OpenShiftClusterDocument, subscriptionDoc *api.SubscriptionDocument) (Manager, error) {
+func NewManager(log *logrus.Entry, _env env.Interface, db database.OpenShiftClusters, cipher encryption.Cipher, billing billing.Manager, doc *api.OpenShiftClusterDocument, subscriptionDoc *api.SubscriptionDocument) (*Manager, error) {
 	localFPAuthorizer, err := _env.FPAuthorizer(_env.TenantID(), azure.PublicCloud.ResourceManagerEndpoint)
 	if err != nil {
 		return nil, err
@@ -88,7 +77,7 @@ func NewManager(log *logrus.Entry, _env env.Interface, db database.OpenShiftClus
 		return nil, err
 	}
 
-	m := &manager{
+	m := &Manager{
 		log:          log,
 		env:          _env,
 		db:           db,
diff --git a/pkg/backend/openshiftcluster/update.go b/pkg/backend/openshiftcluster/update.go
index ebfba9b56..68f46631b 100644
--- a/pkg/backend/openshiftcluster/update.go
+++ b/pkg/backend/openshiftcluster/update.go
@@ -7,7 +7,7 @@ import (
 	"context"
 )
 
-func (m *manager) Update(ctx context.Context) error {
+func (m *Manager) Update(ctx context.Context) error {
 	// TODO: m.ocDynamicValidator.Dynamic is not called because it should run on
 	// an enriched oc.  Neither are we enriching oc here currently, nor does
 	// Dynamic() support running on an enriched oc.
diff --git a/pkg/backend/subscriptions.go b/pkg/backend/subscriptions.go
index b3740fdd8..ebb097c49 100644
--- a/pkg/backend/subscriptions.go
+++ b/pkg/backend/subscriptions.go
@@ -19,10 +19,6 @@ type subscriptionBackend struct {
 	*backend
 }
 
-func newSubscriptionBackend(b *backend) *subscriptionBackend {
-	return &subscriptionBackend{backend: b}
-}
-
 // try tries to dequeue an SubscriptionDocument for work, and works it on a new
 // goroutine.  It returns a boolean to the caller indicating whether it
 // succeeded in dequeuing anything - if this is false, the caller should sleep
diff --git a/pkg/deploy/predeploy.go b/pkg/deploy/predeploy.go
index 7abe35654..77e8041fc 100644
--- a/pkg/deploy/predeploy.go
+++ b/pkg/deploy/predeploy.go
@@ -12,7 +12,6 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault"
 	mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
-	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/to"
 
 	"github.com/Azure/ARO-RP/pkg/deploy/generator"
@@ -176,28 +175,14 @@ func (d *deployer) deployGlobalSubscription(ctx context.Context) error {
 	parameters := d.getParameters(template["parameters"].(map[string]interface{}))
 
 	d.log.Infof("deploying %s", deploymentName)
-	for i := 0; i < 2; i++ {
-		err = d.globaldeployments.CreateOrUpdateAtSubscriptionScopeAndWait(ctx, deploymentName, mgmtfeatures.Deployment{
-			Properties: &mgmtfeatures.DeploymentProperties{
-				Template:   template,
-				Mode:       mgmtfeatures.Incremental,
-				Parameters: parameters.Parameters,
-			},
-			Location: to.StringPtr("centralus"),
-		})
-		if serviceErr, ok := err.(*azure.ServiceError); ok &&
-			serviceErr.Code == "RoleDefinitionUpdateConflict" &&
-			i == 0 {
-			d.log.Print(err)
-			continue
-		}
-		if err != nil {
-			return err
-		}
-
-		break
-	}
-	return nil
+	return d.globaldeployments.CreateOrUpdateAtSubscriptionScopeAndWait(ctx, deploymentName, mgmtfeatures.Deployment{
+		Properties: &mgmtfeatures.DeploymentProperties{
+			Template:   template,
+			Mode:       mgmtfeatures.Incremental,
+			Parameters: parameters.Parameters,
+		},
+		Location: to.StringPtr("centralus"),
+	})
 }
 
 func (d *deployer) deploySubscription(ctx context.Context) error {
diff --git a/pkg/util/dynamichelper/discovery/cache_fallback_discovery_client.go b/pkg/util/dynamichelper/discovery/cache_fallback_discovery_client.go
index fb592bc42..ce8d97887 100644
--- a/pkg/util/dynamichelper/discovery/cache_fallback_discovery_client.go
+++ b/pkg/util/dynamichelper/discovery/cache_fallback_discovery_client.go
@@ -39,7 +39,7 @@ func (d *cacheFallbackDiscoveryClient) ServerResourcesForGroupVersion(groupVersi
 		if cacheErr == nil {
 			return cachedResources, nil
 		}
-		d.log.Info(cacheErr)
+		d.log.Warnf("discovery cache failed. Fallback to live dynamic client. Error: %s", cacheErr)
 	}
 
 	return liveResources, err
diff --git a/pkg/util/dynamichelper/dynamichelper.go b/pkg/util/dynamichelper/dynamichelper.go
index b522d48b3..54a35e99f 100644
--- a/pkg/util/dynamichelper/dynamichelper.go
+++ b/pkg/util/dynamichelper/dynamichelper.go
@@ -66,6 +66,13 @@ func (dh *dynamicHelper) RefreshAPIResources() error {
 	cli = kadiscovery.NewCacheFallbackDiscoveryClient(dh.log, cli)
 
 	_, dh.apiresources, err = cli.ServerGroupsAndResources()
+	if discovery.IsGroupDiscoveryFailedError(err) {
+		// Some group discovery failed; dh.apiresources will have all the ones
+		// that worked. This error can happen with a misconfigured apiservice,
+		// for example. Log it and try to keep going.
+		dh.log.Warn(err)
+		return nil
+	}
 	return err
 }
 
diff --git a/pkg/util/mocks/openshiftcluster/openshiftcluster.go b/pkg/util/mocks/openshiftcluster/openshiftcluster.go
deleted file mode 100644
index eb6ae84cb..000000000
--- a/pkg/util/mocks/openshiftcluster/openshiftcluster.go
+++ /dev/null
@@ -1,91 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/backend/openshiftcluster (interfaces: Manager)
-
-// Package mock_openshiftcluster is a generated GoMock package.
-package mock_openshiftcluster
-
-import (
-	context "context"
-	reflect "reflect"
-
-	gomock "github.com/golang/mock/gomock"
-)
-
-// MockManager is a mock of Manager interface
-type MockManager struct {
-	ctrl     *gomock.Controller
-	recorder *MockManagerMockRecorder
-}
-
-// MockManagerMockRecorder is the mock recorder for MockManager
-type MockManagerMockRecorder struct {
-	mock *MockManager
-}
-
-// NewMockManager creates a new mock instance
-func NewMockManager(ctrl *gomock.Controller) *MockManager {
-	mock := &MockManager{ctrl: ctrl}
-	mock.recorder = &MockManagerMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockManager) EXPECT() *MockManagerMockRecorder {
-	return m.recorder
-}
-
-// AdminUpdate mocks base method
-func (m *MockManager) AdminUpdate(arg0 context.Context) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "AdminUpdate", arg0)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// AdminUpdate indicates an expected call of AdminUpdate
-func (mr *MockManagerMockRecorder) AdminUpdate(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdminUpdate", reflect.TypeOf((*MockManager)(nil).AdminUpdate), arg0)
-}
-
-// Create mocks base method
-func (m *MockManager) Create(arg0 context.Context) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Create", arg0)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Create indicates an expected call of Create
-func (mr *MockManagerMockRecorder) Create(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockManager)(nil).Create), arg0)
-}
-
-// Delete mocks base method
-func (m *MockManager) Delete(arg0 context.Context) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Delete", arg0)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Delete indicates an expected call of Delete
-func (mr *MockManagerMockRecorder) Delete(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockManager)(nil).Delete), arg0)
-}
-
-// Update mocks base method
-func (m *MockManager) Update(arg0 context.Context) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Update", arg0)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Update indicates an expected call of Update
-func (mr *MockManagerMockRecorder) Update(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockManager)(nil).Update), arg0)
-}


Test patch:
diff --git a/pkg/backend/openshiftcluster_test.go b/pkg/backend/openshiftcluster_test.go
deleted file mode 100644
index eaf25ee72..000000000
--- a/pkg/backend/openshiftcluster_test.go
+++ /dev/null
@@ -1,334 +0,0 @@
-package backend
-
-// Copyright (c) Microsoft Corporation.
-// Licensed under the Apache License 2.0.
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/golang/mock/gomock"
-	"github.com/sirupsen/logrus"
-
-	"github.com/Azure/ARO-RP/pkg/api"
-	"github.com/Azure/ARO-RP/pkg/backend/openshiftcluster"
-	"github.com/Azure/ARO-RP/pkg/database"
-	"github.com/Azure/ARO-RP/pkg/env"
-	"github.com/Azure/ARO-RP/pkg/metrics/noop"
-	"github.com/Azure/ARO-RP/pkg/util/billing"
-	"github.com/Azure/ARO-RP/pkg/util/deployment"
-	"github.com/Azure/ARO-RP/pkg/util/encryption"
-	mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env"
-	mock_openshiftcluster "github.com/Azure/ARO-RP/pkg/util/mocks/openshiftcluster"
-	testdb "github.com/Azure/ARO-RP/test/database"
-	testlog "github.com/Azure/ARO-RP/test/util/log"
-)
-
-type backendTestStruct struct {
-	name            string
-	clusterDoc      *api.OpenShiftClusterDocument
-	subscriptionDoc *api.SubscriptionDocument
-	mocks           func(*mock_openshiftcluster.MockManager, database.OpenShiftClusters)
-	fixture         func(*testdb.Fixture)
-	checker         func(*testdb.Checker)
-}
-
-func TestBackendTry(t *testing.T) {
-	mockSubID := "00000000-0000-0000-0000-000000000000"
-	resourceID := fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID)
-
-	for _, tt := range []backendTestStruct{
-		{
-			name: "StateCreating success that sets an InstallPhase stays it in Creating",
-			fixture: func(f *testdb.Fixture) {
-				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState: api.ProvisioningStateCreating,
-						},
-					},
-				})
-				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
-					ID: mockSubID,
-				})
-			},
-			checker: func(c *testdb.Checker) {
-				c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState: api.ProvisioningStateCreating,
-							Install: &api.Install{
-								Phase: api.InstallPhaseBootstrap,
-							},
-						},
-					},
-				})
-			},
-			mocks: func(manager *mock_openshiftcluster.MockManager, dbOpenShiftClusters database.OpenShiftClusters) {
-				manager.EXPECT().Create(gomock.Any()).DoAndReturn(func(ctx context.Context) error {
-					_, err := dbOpenShiftClusters.Patch(ctx, strings.ToLower(resourceID), func(inFlightDoc *api.OpenShiftClusterDocument) error {
-						inFlightDoc.OpenShiftCluster.Properties.Install = &api.Install{}
-						return nil
-					})
-					return err
-				})
-			},
-		},
-		{
-			name: "StateCreating success without an InstallPhase marks provisioning as succeeded",
-			fixture: func(f *testdb.Fixture) {
-				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState: api.ProvisioningStateCreating,
-						},
-					},
-				})
-				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
-					ID: mockSubID,
-				})
-			},
-			checker: func(c *testdb.Checker) {
-				c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState: api.ProvisioningStateSucceeded,
-						},
-					},
-				})
-			},
-			mocks: func(manager *mock_openshiftcluster.MockManager, dbOpenShiftClusters database.OpenShiftClusters) {
-				manager.EXPECT().Create(gomock.Any()).DoAndReturn(func(ctx context.Context) error {
-					_, err := dbOpenShiftClusters.Patch(ctx, strings.ToLower(resourceID), func(inFlightDoc *api.OpenShiftClusterDocument) error {
-						inFlightDoc.OpenShiftCluster.Properties.Install = nil
-						return nil
-					})
-					return err
-				})
-			},
-		},
-		{
-			name: "StateCreating that fails marks ProvisioningState as Failed",
-			fixture: func(f *testdb.Fixture) {
-				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState: api.ProvisioningStateCreating,
-						},
-					},
-				})
-				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
-					ID: mockSubID,
-				})
-			},
-			checker: func(c *testdb.Checker) {
-				c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key:      strings.ToLower(resourceID),
-					Dequeues: 1,
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState:       api.ProvisioningStateFailed,
-							FailedProvisioningState: api.ProvisioningStateCreating,
-						},
-					},
-				})
-			},
-			mocks: func(manager *mock_openshiftcluster.MockManager, dbOpenShiftClusters database.OpenShiftClusters) {
-				manager.EXPECT().Create(gomock.Any()).DoAndReturn(func(ctx context.Context) error {
-					return errors.New("something bad!")
-				})
-			},
-		},
-		{
-			name: "StateAdminUpdating success sets the last ProvisioningState and clears LastAdminUpdateError",
-			fixture: func(f *testdb.Fixture) {
-				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState:     api.ProvisioningStateAdminUpdating,
-							LastProvisioningState: api.ProvisioningStateSucceeded,
-							LastAdminUpdateError:  "oh no",
-						},
-					},
-				})
-				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
-					ID: mockSubID,
-				})
-			},
-			checker: func(c *testdb.Checker) {
-				c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState: api.ProvisioningStateSucceeded,
-						},
-					},
-				})
-			},
-			mocks: func(manager *mock_openshiftcluster.MockManager, dbOpenShiftClusters database.OpenShiftClusters) {
-				manager.EXPECT().AdminUpdate(gomock.Any()).Return(nil)
-			},
-		},
-		{
-			name: "StateAdminUpdating run failure populates LastAdminUpdateError and restores previous provisioning state + failed provisioning state",
-			fixture: func(f *testdb.Fixture) {
-				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState:       api.ProvisioningStateAdminUpdating,
-							LastProvisioningState:   api.ProvisioningStateSucceeded,
-							FailedProvisioningState: api.ProvisioningStateUpdating,
-						},
-					},
-				})
-				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
-					ID: mockSubID,
-				})
-			},
-			checker: func(c *testdb.Checker) {
-				c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState:       api.ProvisioningStateSucceeded,
-							FailedProvisioningState: api.ProvisioningStateUpdating,
-							LastAdminUpdateError:    "oh no!",
-						},
-					},
-				})
-			},
-			mocks: func(manager *mock_openshiftcluster.MockManager, dbOpenShiftClusters database.OpenShiftClusters) {
-				manager.EXPECT().AdminUpdate(gomock.Any()).Return(errors.New("oh no!"))
-			},
-		},
-		{
-			name: "StateDeleting success deletes the document",
-			fixture: func(f *testdb.Fixture) {
-				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
-					Key: strings.ToLower(resourceID),
-					OpenShiftCluster: &api.OpenShiftCluster{
-						ID:       resourceID,
-						Name:     "resourceName",
-						Type:     "Microsoft.RedHatOpenShift/OpenShiftClusters",
-						Location: "location",
-						Properties: api.OpenShiftClusterProperties{
-							ProvisioningState: api.ProvisioningStateDeleting,
-						},
-					},
-				})
-				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
-					ID: mockSubID,
-				})
-			},
-			checker: func(c *testdb.Checker) {},
-			mocks: func(manager *mock_openshiftcluster.MockManager, dbOpenShiftClusters database.OpenShiftClusters) {
-				manager.EXPECT().Delete(gomock.Any()).Return(nil)
-			},
-		},
-	} {
-		t.Run(tt.name, func(t *testing.T) {
-			ctx := context.Background()
-			_, log := testlog.New()
-
-			controller := gomock.NewController(t)
-			defer controller.Finish()
-			manager := mock_openshiftcluster.NewMockManager(controller)
-			_env := mock_env.NewMockInterface(controller)
-			_env.EXPECT().DeploymentMode().Return(deployment.Development)
-
-			dbOpenShiftClusters, clientOpenShiftClusters := testdb.NewFakeOpenShiftClusters()
-			dbSubscriptions, _ := testdb.NewFakeSubscriptions()
-
-			f := testdb.NewFixture().WithOpenShiftClusters(dbOpenShiftClusters).WithSubscriptions(dbSubscriptions)
-			tt.mocks(manager, dbOpenShiftClusters)
-			tt.fixture(f)
-			err := f.Create()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			createManager := func(*logrus.Entry, env.Interface, database.OpenShiftClusters, encryption.Cipher, billing.Manager, *api.OpenShiftClusterDocument, *api.SubscriptionDocument) (openshiftcluster.Manager, error) {
-				return manager, nil
-			}
-
-			b, err := newBackend(ctx, log, _env, nil, nil, dbOpenShiftClusters, dbSubscriptions, nil, &noop.Noop{})
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			b.ocb = &openShiftClusterBackend{
-				backend:    b,
-				newManager: createManager,
-			}
-
-			worked, err := b.ocb.try(ctx)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if !worked {
-				t.Fatal("didnt do work")
-			}
-
-			// wait on the workers to finish their tasks
-			b.waitForWorkerCompletion()
-
-			c := testdb.NewChecker()
-			tt.checker(c)
-
-			errs := c.CheckOpenShiftClusters(clientOpenShiftClusters)
-			for _, err := range errs {
-				t.Error(err)
-			}
-		})
-	}
-}