Repository: Azure/ARO-RP
Pull request #1413 (2021-03-24 18:43:14)

Remove persistence prometheus
### Which issue this PR addresses:

<!--
Please include a link to the VSTS work item as well as any GitHub issues.

Usage: `Fixes #<GitHub issue number>`, or `Fixes (paste link of issue)`.
-->
Fixes [8964694](https://msazure.visualstudio.com/AzureRedHatOpenShift/_workitems/edit/8964696/)

### What this PR does / why we need it:

Removes the persistent volume configuration for prometheus by not allowing the setting to be triggered by the feature flag in the operator.  This will disable persistent prometheus across the fleet of clusters.
<!--
Include a brief summary of what the PR is intended to accomplish and how the PR
does it. (2-3 sentences)
-->

### Test plan for issue:

<!--
How did you test that this PR works?

- Are there unit tests?
- Are there integration/e2e tests?
- If it is not possible to write automated tests, explain why and document how
  to manually test and verify the feature.
-->
Modified unit tests that reflect the disabling of the prometheus persistence.  

### Is there any documentation that needs to be updated for this PR?

<!--
- If yes and the docs are in GitHub, include doc updates in the PR.
- If yes and the docs are not in GitHub (i.e. VSTS wiki), include a link to the
  docs.
- If no, explain why (e.g. "tech debt cleanup, N/A").
-->
`None`



Patch:
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 49c4e29b2..a2cd6809c 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,7 +1,7 @@
 ### Which issue this PR addresses:
 
 <!--
-Please include a link to the VSTS work item as well as any GitHub issues.
+Please include a link to the ADO work item as well as any GitHub issues.
 
 Usage: `Fixes #<GitHub issue number>`, or `Fixes (paste link of issue)`.
 -->
@@ -29,7 +29,7 @@ How did you test that this PR works?
 
 <!--
 - If yes and the docs are in GitHub, include doc updates in the PR.
-- If yes and the docs are not in GitHub (i.e. VSTS wiki), include a link to the
+- If yes and the docs are not in GitHub (i.e. ADO wiki), include a link to the
   docs.
 - If no, explain why (e.g. "tech debt cleanup, N/A").
 -->
diff --git a/Makefile b/Makefile
index f47ad6b86..37ff0c9d6 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 SHELL = /bin/bash
-COMMIT = $(shell git rev-parse --short HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty)
+COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty)
 ARO_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/aro:$(COMMIT)
 
 ifneq ($(shell uname -s),Darwin)
diff --git a/cmd/aro/main.go b/cmd/aro/main.go
index b6dc42dca..7cac74bd3 100644
--- a/cmd/aro/main.go
+++ b/cmd/aro/main.go
@@ -62,7 +62,7 @@ func main() {
 		err = rp(ctx, log, audit)
 	case "portal":
 		checkArgs(1)
-		err = portal(ctx, log)
+		err = portal(ctx, log, audit)
 	case "operator":
 		checkArgs(2)
 		err = operator(ctx, log)
diff --git a/cmd/aro/operator.go b/cmd/aro/operator.go
index 801288a1c..8cd98bdcf 100644
--- a/cmd/aro/operator.go
+++ b/cmd/aro/operator.go
@@ -24,6 +24,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/operator/controllers"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/alertwebhook"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/checker"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers/clusteroperatoraro"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/dnsmasq"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/genevalogging"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/monitoring"
@@ -100,6 +101,11 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 			restConfig)).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("unable to create controller Genevalogging: %v", err)
 		}
+		if err = (clusteroperatoraro.NewReconciler(
+			log.WithField("controller", controllers.ClusterOperatorAROName),
+			arocli, configcli)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller ClusterOperatorARO: %v", err)
+		}
 		if err = (pullsecret.NewReconciler(
 			log.WithField("controller", controllers.PullSecretControllerName),
 			kubernetescli)).SetupWithManager(mgr); err != nil {
@@ -154,7 +160,7 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 
 	if err = (checker.NewReconciler(
 		log.WithField("controller", controllers.CheckerControllerName),
-		maocli, arocli, role, isDevelopmentMode)).SetupWithManager(mgr); err != nil {
+		maocli, arocli, kubernetescli, role, isDevelopmentMode)).SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("unable to create controller InternetChecker: %v", err)
 	}
 
diff --git a/cmd/aro/portal.go b/cmd/aro/portal.go
index 975cfe90e..696999d64 100644
--- a/cmd/aro/portal.go
+++ b/cmd/aro/portal.go
@@ -24,7 +24,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/keyvault"
 )
 
-func portal(ctx context.Context, log *logrus.Entry) error {
+func portal(ctx context.Context, log *logrus.Entry, audit *logrus.Entry) error {
 	_env, err := env.NewCore(ctx, log)
 	if err != nil {
 		return err
@@ -166,7 +166,7 @@ func portal(ctx context.Context, log *logrus.Entry) error {
 
 	log.Print("listening")
 
-	p := pkgportal.NewPortal(_env, log.WithField("component", "portal"), log.WithField("component", "portal-access"), l, sshl, verifier, hostname, servingKey, servingCerts, clientID, clientKey, clientCerts, sessionKey, sshKey, groupIDs, elevatedGroupIDs, dbOpenShiftClusters, dbPortal, dialer)
+	p := pkgportal.NewPortal(_env, audit, log.WithField("component", "portal"), log.WithField("component", "portal-access"), l, sshl, verifier, hostname, servingKey, servingCerts, clientID, clientKey, clientCerts, sessionKey, sshKey, groupIDs, elevatedGroupIDs, dbOpenShiftClusters, dbPortal, dialer)
 
 	return p.Run(ctx)
 }
diff --git a/docs/deploy-development-rp.md b/docs/deploy-development-rp.md
index d2fa8daac..716916121 100644
--- a/docs/deploy-development-rp.md
+++ b/docs/deploy-development-rp.md
@@ -206,12 +206,27 @@
 
 * "SSH" to a cluster node:
 
-  * First, get the admin kubeconfig and `export KUBECONFIG` as detailed above.
-
-  ```bash
-  CLUSTER=cluster hack/ssh-agent.sh [master-{0,1,2}]
-  ```
-
+  * Get the admin kubeconfig and `export KUBECONFIG` as detailed above.
+  * Run the ssh-agent.sh script. This takes the argument is the name of the NIC attached to the VM you are trying to ssh to.
+   * Given the following nodes these commands would be used to connect to the respective node
+
+    ```
+   $ oc get nodes
+   NAME                                     STATUS     ROLES    AGE   VERSION
+   aro-dev-abc123-master-0               Ready      master   47h   v1.19.0+2f3101c
+   aro-dev-abc123-master-1               Ready      master   47h   v1.19.0+2f3101c
+   aro-dev-abc123-master-2               Ready      master   47h   v1.19.0+2f3101c
+   aro-dev-abc123-worker-eastus1-2s5rb   Ready      worker   47h   v1.19.0+2f3101c
+   aro-dev-abc123-worker-eastus2-php82   Ready      worker   47h   v1.19.0+2f3101c
+   aro-dev-abc123-worker-eastus3-cbqs2   Ready      worker   47h   v1.19.0+2f3101c
+
+
+   CLUSTER=cluster hack/ssh-agent.sh master0 # master node aro-dev-abc123-master-0
+   CLUSTER=cluster hack/ssh-agent.sh aro-dev-abc123-worker-eastus1-2s5rb # worker aro-dev-abc123-worker-eastus1-2s5rb
+   CLUSTER=cluster hack/ssh-agent.sh eastus1 # worker aro-dev-abc123-worker-eastus1-2s5rb
+   CLUSTER=cluster hack/ssh-agent.sh 2s5rb  # worker aro-dev-abc123-worker-eastus1-2s5rb
+   CLUSTER=cluster hack/ssh-agent.sh bootstrap # the bootstrap node used to provision cluster
+   ```
 
 ### Metrics
 
diff --git a/go.mod b/go.mod
index 33ffcb3a2..a66891b14 100644
--- a/go.mod
+++ b/go.mod
@@ -70,6 +70,7 @@ require (
 	github.com/openshift/cluster-api-provider-baremetal v0.0.0-20210126223742-4e254d08e0ad // indirect
 	github.com/openshift/console-operator v0.0.0-20210216151626-6e1cbc849915
 	github.com/openshift/installer v0.16.1
+	github.com/openshift/library-go v0.0.0-20200909173121-1d055d971916
 	github.com/openshift/machine-api-operator v0.2.1-0.20210104142355-8e6ae0acdfcf
 	github.com/openshift/machine-config-operator v4.2.0-alpha.0.0.20190917115525-033375cbe820+incompatible
 	github.com/operator-framework/operator-sdk v1.4.2
diff --git a/go.sum b/go.sum
index d20a4bbad..61549876e 100644
--- a/go.sum
+++ b/go.sum
@@ -1657,6 +1657,7 @@ github.com/openshift/library-go v0.0.0-20191003152030-97c62d8a2901/go.mod h1:NBt
 github.com/openshift/library-go v0.0.0-20200324092245-db2a8546af81/go.mod h1:Qc5duoXHzAKyUfA0REIlG/rdfWzknOpp9SiDiyg5Y7A=
 github.com/openshift/library-go v0.0.0-20200831114015-2ab0c61c15de/go.mod h1:6vwp+YhYOIlj8MpkQKkebTTSn2TuYyvgiAFQ206jIEQ=
 github.com/openshift/library-go v0.0.0-20200907120738-ea57b121ba1a/go.mod h1:6vwp+YhYOIlj8MpkQKkebTTSn2TuYyvgiAFQ206jIEQ=
+github.com/openshift/library-go v0.0.0-20200909173121-1d055d971916 h1:H9XwZlu78Pn87KNBGGjZY0L2fLOoPnzEajigVur5ZOY=
 github.com/openshift/library-go v0.0.0-20200909173121-1d055d971916/go.mod h1:6vwp+YhYOIlj8MpkQKkebTTSn2TuYyvgiAFQ206jIEQ=
 github.com/openshift/machine-api-operator v0.2.1-0.20210212025836-cb508cd8777d h1:HPsOufOinE9dt/EtqY20gigk+es+y53GtcQPj4ItRl0=
 github.com/openshift/machine-api-operator v0.2.1-0.20210212025836-cb508cd8777d/go.mod h1:UZK7t/t3dO73RqC9MsXM/BAIMWRsNaE9NEKUXBBY8y8=
diff --git a/hack/ssh-agent.sh b/hack/ssh-agent.sh
index 898207ec9..16fb44718 100755
--- a/hack/ssh-agent.sh
+++ b/hack/ssh-agent.sh
@@ -7,11 +7,11 @@
 usage() {
     echo "usage: CLUSTER=cluster $0 hostname_pattern" >&2
     echo "       Examples: CLUSTER=cluster $0 master1" >&2
+    echo "                 CLUSTER=cluster $0 eastus1 # worker node 1" >&2
     echo "                 CLUSTER=cluster $0 bootstrap" >&2
     exit 1
 }
 
-
 if [[ "$#" -ne 1 ]]; then
    usage
 fi
@@ -24,12 +24,12 @@ trap cleanup EXIT
 
 eval "$(ssh-agent | grep -v '^echo ')"
 
-if [[ -z "$CLUSTER" ]]; then
-    echo "CLUSTER must be specified"
-    usage
-fi
-
 if [[ -z "$RESOURCEID" ]]; then
+    if [[ -z "$CLUSTER" ]]; then
+        echo "CLUSTER must be specified"
+        usage
+    fi
+    
     RESOURCEID="/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${RESOURCEGROUP}/providers/Microsoft.RedHatOpenShift/openShiftClusters/${CLUSTER}"
 fi
 
diff --git a/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go b/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go
index 4782d0039..31bd905f3 100644
--- a/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go
+++ b/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go
@@ -13,17 +13,18 @@ const (
 	InternetReachableFromMaster status.ConditionType = "InternetReachableFromMaster"
 	InternetReachableFromWorker status.ConditionType = "InternetReachableFromWorker"
 	MachineValid                status.ConditionType = "MachineValid"
+	ServicePrincipalValid       status.ConditionType = "ServicePrincipalValid"
 )
 
 // AllConditionTypes is a operator conditions currently in use, any condition not in this list is not
 // added to the operator.status.conditions list
 func AllConditionTypes() []status.ConditionType {
-	return []status.ConditionType{InternetReachableFromMaster, InternetReachableFromWorker, MachineValid}
+	return []status.ConditionType{InternetReachableFromMaster, InternetReachableFromWorker, MachineValid, ServicePrincipalValid}
 }
 
 // ClusterChecksTypes represents checks performed on the cluster to verify basic functionality
 func ClusterChecksTypes() []status.ConditionType {
-	return []status.ConditionType{InternetReachableFromMaster, InternetReachableFromWorker, MachineValid}
+	return []status.ConditionType{InternetReachableFromMaster, InternetReachableFromWorker, MachineValid, ServicePrincipalValid}
 }
 
 type GenevaLoggingSpec struct {
diff --git a/pkg/operator/controllers/checker/checker_controller.go b/pkg/operator/controllers/checker/checker_controller.go
index c94e81bb8..c3840f5b8 100644
--- a/pkg/operator/controllers/checker/checker_controller.go
+++ b/pkg/operator/controllers/checker/checker_controller.go
@@ -12,6 +12,7 @@ import (
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -32,11 +33,14 @@ type CheckerController struct {
 	checkers []Checker
 }
 
-func NewReconciler(log *logrus.Entry, maocli maoclient.Interface, arocli aroclient.Interface, role string, isDevelopmentMode bool) *CheckerController {
+func NewReconciler(log *logrus.Entry, maocli maoclient.Interface, arocli aroclient.Interface, kubernetescli kubernetes.Interface, role string, isDevelopmentMode bool) *CheckerController {
 	checkers := []Checker{NewInternetChecker(log, arocli, role)}
 
 	if role == operator.RoleMaster {
-		checkers = append(checkers, NewMachineChecker(log, maocli, arocli, role, isDevelopmentMode))
+		checkers = append(checkers,
+			NewMachineChecker(log, maocli, arocli, role, isDevelopmentMode),
+			NewServicePrincipalChecker(log, maocli, arocli, kubernetescli, role),
+		)
 	}
 
 	return &CheckerController{
diff --git a/pkg/operator/controllers/checker/const.go b/pkg/operator/controllers/checker/const.go
new file mode 100644
index 000000000..a4f70c304
--- /dev/null
+++ b/pkg/operator/controllers/checker/const.go
@@ -0,0 +1,10 @@
+package checker
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+const (
+	azureCredentialSecretNamespace = "kube-system"
+	azureCredentialSecretName      = "azure-credentials"
+	machineSetsNamespace           = "openshift-machine-api"
+)
diff --git a/pkg/operator/controllers/checker/machinechecker.go b/pkg/operator/controllers/checker/machinechecker.go
index a28ba904f..08c5dc499 100644
--- a/pkg/operator/controllers/checker/machinechecker.go
+++ b/pkg/operator/controllers/checker/machinechecker.go
@@ -25,10 +25,6 @@ import (
 	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
 )
 
-const (
-	machineSetsNamespace = "openshift-machine-api"
-)
-
 // MachineChecker reconciles the alertmanager webhook
 type MachineChecker struct {
 	clustercli        maoclient.Interface
diff --git a/pkg/operator/controllers/checker/serviceprincipalchecker.go b/pkg/operator/controllers/checker/serviceprincipalchecker.go
new file mode 100644
index 000000000..3e2b1f680
--- /dev/null
+++ b/pkg/operator/controllers/checker/serviceprincipalchecker.go
@@ -0,0 +1,121 @@
+package checker
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/go-autorest/autorest/azure"
+	maoclient "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned"
+	"github.com/operator-framework/operator-sdk/pkg/status"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/api/validate/dynamic"
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers"
+	"github.com/Azure/ARO-RP/pkg/util/aad"
+)
+
+type ServicePrincipalChecker struct {
+	log           *logrus.Entry
+	clustercli    maoclient.Interface
+	arocli        aroclient.Interface
+	kubernetescli kubernetes.Interface
+	role          string
+}
+
+func NewServicePrincipalChecker(log *logrus.Entry, maocli maoclient.Interface, arocli aroclient.Interface, kubernetescli kubernetes.Interface, role string) *ServicePrincipalChecker {
+	return &ServicePrincipalChecker{
+		log:           log,
+		clustercli:    maocli,
+		arocli:        arocli,
+		kubernetescli: kubernetescli,
+		role:          role,
+	}
+}
+
+func (r *ServicePrincipalChecker) Name() string {
+	return "ServicePrincipalChecker"
+}
+
+func (r *ServicePrincipalChecker) Check(ctx context.Context) error {
+	cond := &status.Condition{
+		Type:    arov1alpha1.ServicePrincipalValid,
+		Status:  corev1.ConditionTrue,
+		Message: "service principal is valid",
+		Reason:  "CheckDone",
+	}
+
+	cluster, err := r.arocli.AroV1alpha1().Clusters().Get(ctx, arov1alpha1.SingletonClusterName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	resource, err := azure.ParseResourceID(cluster.Spec.ResourceID)
+	if err != nil {
+		return err
+	}
+
+	azEnv, err := azure.EnvironmentFromName(cluster.Spec.AZEnvironment)
+	if err != nil {
+		return err
+	}
+
+	azCred, err := azCredentials(ctx, r.kubernetescli)
+	if err != nil {
+		return err
+	}
+
+	_, err = aad.GetToken(ctx, r.log, azCred.clientID, azCred.clientSecret, azCred.tenantID, azEnv.ActiveDirectoryEndpoint, azEnv.ResourceManagerEndpoint)
+	if err != nil {
+		updateFailedCondition(cond, err)
+	}
+
+	spDynamic, err := dynamic.NewValidator(r.log, &azEnv, resource.SubscriptionID, nil, dynamic.AuthorizerClusterServicePrincipal)
+	if err != nil {
+		return err
+	}
+
+	err = spDynamic.ValidateServicePrincipal(ctx, azCred.clientID, azCred.clientSecret, azCred.tenantID)
+	if err != nil {
+		updateFailedCondition(cond, err)
+	}
+
+	return controllers.SetCondition(ctx, r.arocli, cond, r.role)
+}
+
+type credentials struct {
+	clientID     string
+	clientSecret string
+	tenantID     string
+}
+
+func azCredentials(ctx context.Context, kubernetescli kubernetes.Interface) (*credentials, error) {
+	var creds credentials
+
+	mysec, err := kubernetescli.CoreV1().Secrets(azureCredentialSecretNamespace).Get(ctx, azureCredentialSecretName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	creds.clientID = string(mysec.Data["azure_client_id"])
+	creds.clientSecret = string(mysec.Data["azure_client_secret"])
+	creds.tenantID = string(mysec.Data["azure_tenant_id"])
+
+	return &creds, nil
+}
+
+func updateFailedCondition(cond *status.Condition, err error) {
+	cond.Status = corev1.ConditionFalse
+	if tErr, ok := err.(*api.CloudError); ok {
+		cond.Message = tErr.Message
+	} else {
+		cond.Message = err.Error()
+	}
+}
diff --git a/pkg/operator/controllers/clusteroperatoraro/clusteroperatoraro_controller.go b/pkg/operator/controllers/clusteroperatoraro/clusteroperatoraro_controller.go
new file mode 100644
index 000000000..c547ced37
--- /dev/null
+++ b/pkg/operator/controllers/clusteroperatoraro/clusteroperatoraro_controller.go
@@ -0,0 +1,183 @@
+package clusteroperatoraro
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	configv1 "github.com/openshift/api/config/v1"
+	configclient "github.com/openshift/client-go/config/clientset/versioned"
+	"github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers"
+	"github.com/sirupsen/logrus"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/util/retry"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+	aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers"
+)
+
+const (
+	clusterOperatorName = "aro"
+)
+
+// The default set of status change reasons.
+const (
+	reasonAsExpected   = "AsExpected"
+	reasonInitializing = "Initializing"
+)
+
+type ClusterOperatorAROReconciler struct {
+	// Replace configcli with CO client
+	configcli configclient.Interface
+	arocli    aroclient.Interface
+	log       *logrus.Entry
+}
+
+func NewReconciler(log *logrus.Entry, arocli aroclient.Interface, configcli configclient.Interface) *ClusterOperatorAROReconciler {
+	return &ClusterOperatorAROReconciler{
+		configcli: configcli,
+		arocli:    arocli,
+		log:       log,
+	}
+}
+
+// SetupWithManager setup our manager
+func (r *ClusterOperatorAROReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	aroClusterPredicate := predicate.NewPredicateFuncs(func(meta metav1.Object, object runtime.Object) bool {
+		return meta.GetName() == arov1alpha1.SingletonClusterName
+	})
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&arov1alpha1.Cluster{}, builder.WithPredicates(aroClusterPredicate)).
+		Owns(&configv1.ClusterOperator{}).
+		Named(controllers.ClusterOperatorAROName).
+		Complete(r)
+}
+
+func (r *ClusterOperatorAROReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
+	// TODO(mj): Reconcile will eventually be receiving a ctx (https://github.com/kubernetes-sigs/controller-runtime/blob/7ef2da0bc161d823f084ad21ff5f9c9bd6b0cc39/pkg/reconcile/reconcile.go#L93)
+	ctx := context.TODO()
+
+	cluster, err := r.arocli.AroV1alpha1().Clusters().Get(ctx, arov1alpha1.SingletonClusterName, metav1.GetOptions{})
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	return reconcile.Result{}, retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		co, err := r.getOrCreateClusterOperator(ctx)
+		if err != nil {
+			return err
+		}
+
+		err = controllerutil.SetControllerReference(cluster, co, scheme.Scheme)
+		if err != nil {
+			return err
+		}
+
+		co, err = r.configcli.ConfigV1().ClusterOperators().Update(ctx, co, metav1.UpdateOptions{})
+		if err != nil {
+			return err
+		}
+
+		return r.setClusterOperatorStatus(ctx, co)
+	})
+}
+
+func (r *ClusterOperatorAROReconciler) setClusterOperatorStatus(ctx context.Context, co *configv1.ClusterOperator) error {
+	// TODO: Replace with a real conditions based on aro operator conditions
+	currentTime := metav1.Now()
+	conditions := []configv1.ClusterOperatorStatusCondition{
+		{
+			Type:               configv1.OperatorAvailable,
+			Status:             configv1.ConditionTrue,
+			LastTransitionTime: currentTime,
+			Reason:             reasonAsExpected,
+		},
+		{
+			Type:               configv1.OperatorProgressing,
+			Status:             configv1.ConditionFalse,
+			LastTransitionTime: currentTime,
+			Reason:             reasonAsExpected,
+		},
+		{
+			Type:               configv1.OperatorDegraded,
+			Status:             configv1.ConditionFalse,
+			LastTransitionTime: currentTime,
+			Reason:             reasonAsExpected,
+		},
+		{
+			Type:               configv1.OperatorUpgradeable,
+			Status:             configv1.ConditionTrue,
+			LastTransitionTime: currentTime,
+			Reason:             reasonAsExpected,
+		},
+	}
+
+	for _, c := range conditions {
+		v1helpers.SetStatusCondition(&co.Status.Conditions, c)
+	}
+
+	_, err := r.configcli.ConfigV1().ClusterOperators().UpdateStatus(ctx, co, metav1.UpdateOptions{})
+	return err
+}
+
+func (r *ClusterOperatorAROReconciler) getOrCreateClusterOperator(ctx context.Context) (*configv1.ClusterOperator, error) {
+	co, err := r.configcli.ConfigV1().ClusterOperators().Get(ctx, clusterOperatorName, metav1.GetOptions{})
+	if !kerrors.IsNotFound(err) {
+		return co, err
+	}
+
+	r.log.Infof("ClusterOperator does not exist, creating a new one.")
+	defaultCo := r.defaultOperator()
+	co, err = r.configcli.ConfigV1().ClusterOperators().Create(ctx, defaultCo, metav1.CreateOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	co.Status = defaultCo.Status
+
+	return r.configcli.ConfigV1().ClusterOperators().UpdateStatus(ctx, co, metav1.UpdateOptions{})
+}
+
+func (r *ClusterOperatorAROReconciler) defaultOperator() *configv1.ClusterOperator {
+	currentTime := metav1.Now()
+	return &configv1.ClusterOperator{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: clusterOperatorName,
+		},
+		Status: configv1.ClusterOperatorStatus{
+			Conditions: []configv1.ClusterOperatorStatusCondition{
+				{
+					Type:               configv1.OperatorAvailable,
+					Status:             configv1.ConditionFalse,
+					LastTransitionTime: currentTime,
+					Reason:             reasonInitializing,
+					Message:            "Operator is initializing",
+				},
+				{
+					Type:               configv1.OperatorProgressing,
+					Status:             configv1.ConditionTrue,
+					LastTransitionTime: currentTime,
+					Reason:             reasonInitializing,
+					Message:            "Operator is initializing",
+				},
+				{
+					Type:               configv1.OperatorDegraded,
+					Status:             configv1.ConditionFalse,
+					LastTransitionTime: currentTime,
+					Reason:             reasonAsExpected,
+				},
+			},
+		},
+	}
+}
diff --git a/pkg/operator/controllers/const.go b/pkg/operator/controllers/const.go
index 7c7abff47..0f4fb148a 100644
--- a/pkg/operator/controllers/const.go
+++ b/pkg/operator/controllers/const.go
@@ -5,6 +5,7 @@ package controllers
 
 const (
 	AlertwebhookControllerName             = "Alertwebhook"
+	ClusterOperatorAROName                 = "ClusterOperatorARO"
 	GenevaLoggingControllerName            = "GenevaLogging"
 	PullSecretControllerName               = "PullSecret"
 	WorkaroundControllerName               = "Workaround"
diff --git a/pkg/operator/controllers/monitoring/monitoring_controller.go b/pkg/operator/controllers/monitoring/monitoring_controller.go
index dc127c6c4..9fe9808c3 100644
--- a/pkg/operator/controllers/monitoring/monitoring_controller.go
+++ b/pkg/operator/controllers/monitoring/monitoring_controller.go
@@ -84,12 +84,6 @@ func (r *Reconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
 	// TODO(mj): Reconcile will eventually be receiving a ctx (https://github.com/kubernetes-sigs/controller-runtime/blob/7ef2da0bc161d823f084ad21ff5f9c9bd6b0cc39/pkg/reconcile/reconcile.go#L93)
 	ctx := context.TODO()
 
-	// check feature gates and if set to false remove any persistence
-	cluster, err := r.arocli.AroV1alpha1().Clusters().Get(ctx, arov1alpha1.SingletonClusterName, metav1.GetOptions{})
-	if err != nil {
-		return reconcile.Result{}, err
-	}
-
 	return reconcile.Result{}, retry.RetryOnConflict(retry.DefaultRetry, func() error {
 		cm, isCreate, err := r.monitoringConfigMap(ctx)
 		if err != nil {
@@ -111,28 +105,15 @@ func (r *Reconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
 		}
 
 		changed := false
-		switch cluster.Spec.Features.PersistentPrometheus {
-		case true:
-			// we are enabling persistence
-			if configData.PrometheusK8s.Retention != "15d" {
-				configData.PrometheusK8s.Retention = "15d"
-				changed = true
-			}
-			if configData.PrometheusK8s.VolumeClaimTemplate.Spec.Resources.Requests.Storage != "100Gi" {
-				configData.PrometheusK8s.VolumeClaimTemplate.Spec.Resources.Requests.Storage = "100Gi"
-				changed = true
-			}
-			// we are disabling persistence. We use omitempty on the struct to
-			// clean the fields
-		case false:
-			if configData.PrometheusK8s.Retention != "" {
-				configData.PrometheusK8s.Retention = ""
-				changed = true
-			}
-			if configData.PrometheusK8s.VolumeClaimTemplate.Spec.Resources.Requests.Storage != "" {
-				configData.PrometheusK8s.VolumeClaimTemplate.Spec.Resources.Requests.Storage = ""
-				changed = true
-			}
+		// we are disabling persistence. We use omitempty on the struct to
+		// clean the fields
+		if configData.PrometheusK8s.Retention != "" {
+			configData.PrometheusK8s.Retention = ""
+			changed = true
+		}
+		if configData.PrometheusK8s.VolumeClaimTemplate.Spec.Resources.Requests.Storage != "" {
+			configData.PrometheusK8s.VolumeClaimTemplate.Spec.Resources.Requests.Storage = ""
+			changed = true
 		}
 
 		if !isCreate && !changed {
@@ -152,10 +133,10 @@ func (r *Reconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
 		cm.Data["config.yaml"] = string(cmYaml)
 
 		if isCreate {
-			r.log.Infof("re-creating monitoring configmap. featureFlag %t", cluster.Spec.Features.PersistentPrometheus)
+			r.log.Infof("re-creating monitoring configmap. %s", monitoringName.Name)
 			_, err = r.kubernetescli.CoreV1().ConfigMaps(monitoringName.Namespace).Create(ctx, cm, metav1.CreateOptions{})
 		} else {
-			r.log.Infof("updating monitoring configmap. featureFlag %t", cluster.Spec.Features.PersistentPrometheus)
+			r.log.Infof("updating monitoring configmap. %s", monitoringName.Name)
 			_, err = r.kubernetescli.CoreV1().ConfigMaps(monitoringName.Namespace).Update(ctx, cm, metav1.UpdateOptions{})
 		}
 		return err
@@ -185,14 +166,14 @@ func (r *Reconciler) monitoringConfigMap(ctx context.Context) (*corev1.ConfigMap
 func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.log.Info("starting starting cluster monitoring controller")
 
-	monitoringConfigMapPredicate := predicate.NewPredicateFuncs(func(meta metav1.Object, object runtime.Object) bool {
-		return meta.GetName() == monitoringName.Name && meta.GetNamespace() == monitoringName.Namespace
-	})
-
 	aroClusterPredicate := predicate.NewPredicateFuncs(func(meta metav1.Object, object runtime.Object) bool {
 		return meta.GetName() == arov1alpha1.SingletonClusterName
 	})
 
+	monitoringConfigMapPredicate := predicate.NewPredicateFuncs(func(meta metav1.Object, object runtime.Object) bool {
+		return meta.GetName() == monitoringName.Name && meta.GetNamespace() == monitoringName.Namespace
+	})
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&arov1alpha1.Cluster{}, builder.WithPredicates(aroClusterPredicate)).
 		// https://github.com/kubernetes-sigs/controller-runtime/issues/1173
diff --git a/pkg/operator/controllers/workaround/cleanprompvc.go b/pkg/operator/controllers/workaround/cleanprompvc.go
new file mode 100644
index 000000000..91218b1f9
--- /dev/null
+++ b/pkg/operator/controllers/workaround/cleanprompvc.go
@@ -0,0 +1,98 @@
+package workaround
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+// Clean the remaining PVCs in openshift-monitoring namespace.
+// These PVCs with labels: app=prometheus,prometheus=k8s are left
+// behind after switching back to use emptydir as persistent storage
+// for prometheus by disabling featureflag in monitoing controller.
+// This workaround is in effect for all clusters set to
+// have non-persistent prometheus.
+// The cleanup loop removes only up to 2 PVCs as this is the
+// production configuration at the time of the workaround release.
+
+import (
+	"context"
+
+	"github.com/ghodss/yaml"
+	"github.com/sirupsen/logrus"
+	"github.com/ugorji/go/codec"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
+
+	"github.com/Azure/ARO-RP/pkg/operator/controllers/monitoring"
+	"github.com/Azure/ARO-RP/pkg/util/version"
+)
+
+const (
+	prometheusLabels    = "app=prometheus,prometheus=k8s"
+	monitoringName      = "cluster-monitoring-config"
+	monitoringNamespace = "openshift-monitoring"
+)
+
+type cleanPromPVC struct {
+	log *logrus.Entry
+	cli kubernetes.Interface
+}
+
+func NewCleanFromPVCWorkaround(log *logrus.Entry, cli kubernetes.Interface) Workaround {
+	return &cleanPromPVC{
+		log: log,
+		cli: cli,
+	}
+}
+
+func (*cleanPromPVC) Name() string {
+	return "Clean prometheus PVC after disabling persistency"
+}
+
+func (c *cleanPromPVC) IsRequired(clusterVersion *version.Version) bool {
+	cm, err := c.cli.CoreV1().ConfigMaps(monitoringNamespace).Get(context.Background(), monitoringName, metav1.GetOptions{})
+	if err != nil {
+		return false
+	}
+
+	configDataJSON, err := yaml.YAMLToJSON([]byte(cm.Data["config.yaml"]))
+	if err != nil {
+		return false
+	}
+	var configData monitoring.Config
+	handle := new(codec.JsonHandle)
+	err = codec.NewDecoderBytes(configDataJSON, handle).Decode(&configData)
+	if err != nil {
+		return false
+	}
+
+	if configData.PrometheusK8s.Retention == "" && configData.PrometheusK8s.VolumeClaimTemplate.Spec.Resources.Requests.Storage == "" {
+		return true
+	}
+
+	return false
+}
+
+func (c *cleanPromPVC) Ensure(ctx context.Context) error {
+
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		pvcList, err := c.cli.CoreV1().PersistentVolumeClaims(monitoringNamespace).List(ctx, metav1.ListOptions{LabelSelector: prometheusLabels})
+		if err != nil {
+			return err
+		}
+
+		for _, pvc := range pvcList.Items {
+			err = c.cli.CoreV1().PersistentVolumeClaims(monitoringNamespace).Delete(ctx, pvc.Name, metav1.DeleteOptions{})
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+
+	return err
+}
+
+func (c *cleanPromPVC) Remove(ctx context.Context) error {
+	return nil
+}
diff --git a/pkg/operator/controllers/workaround/workaround_controller.go b/pkg/operator/controllers/workaround/workaround_controller.go
index 6f9832f09..ec480bad1 100644
--- a/pkg/operator/controllers/workaround/workaround_controller.go
+++ b/pkg/operator/controllers/workaround/workaround_controller.go
@@ -44,7 +44,7 @@ func NewReconciler(log *logrus.Entry, kubernetescli kubernetes.Interface, config
 		configcli:     configcli,
 		arocli:        arocli,
 		restConfig:    restConfig,
-		workarounds:   []Workaround{NewSystemReserved(log, mcocli, dh), NewIfReload(log, kubernetescli)},
+		workarounds:   []Workaround{NewSystemReserved(log, mcocli, dh), NewIfReload(log, kubernetescli), NewCleanFromPVCWorkaround(log, kubernetescli)},
 		log:           log,
 	}
 }
diff --git a/pkg/portal/kubeconfig/kubeconfig.go b/pkg/portal/kubeconfig/kubeconfig.go
index af4f2951c..503e3980a 100644
--- a/pkg/portal/kubeconfig/kubeconfig.go
+++ b/pkg/portal/kubeconfig/kubeconfig.go
@@ -22,6 +22,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/api/validate"
 	"github.com/Azure/ARO-RP/pkg/database"
+	"github.com/Azure/ARO-RP/pkg/env"
 	"github.com/Azure/ARO-RP/pkg/portal/middleware"
 	"github.com/Azure/ARO-RP/pkg/portal/util/clientcache"
 	"github.com/Azure/ARO-RP/pkg/proxy"
@@ -49,6 +50,8 @@ type kubeconfig struct {
 }
 
 func New(baseLog *logrus.Entry,
+	audit *logrus.Entry,
+	env env.Core,
 	baseAccessLog *logrus.Entry,
 	servingCert *x509.Certificate,
 	elevatedGroupIDs []string,
@@ -83,7 +86,7 @@ func New(baseLog *logrus.Entry,
 
 	bearerAuthenticatedRouter := unauthenticatedRouter.NewRoute().Subrouter()
 	bearerAuthenticatedRouter.Use(middleware.Bearer(k.dbPortal))
-	bearerAuthenticatedRouter.Use(middleware.Log(k.baseAccessLog))
+	bearerAuthenticatedRouter.Use(middleware.Log(env, audit, k.baseAccessLog))
 
 	bearerAuthenticatedRouter.PathPrefix("/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/microsoft.redhatopenshift/openshiftclusters/{resourceName}/kubeconfig/proxy/").Handler(rp)
 
diff --git a/pkg/portal/middleware/aad.go b/pkg/portal/middleware/aad.go
index 33efddc53..160bf1603 100644
--- a/pkg/portal/middleware/aad.go
+++ b/pkg/portal/middleware/aad.go
@@ -106,6 +106,7 @@ type aad struct {
 }
 
 func NewAAD(log *logrus.Entry,
+	audit *logrus.Entry,
 	env env.Core,
 	baseAccessLog *logrus.Entry,
 	hostname string,
@@ -151,7 +152,7 @@ func NewAAD(log *logrus.Entry,
 	a.store.Options.HttpOnly = true
 	a.store.Options.SameSite = http.SameSiteLaxMode
 
-	unauthenticatedRouter.NewRoute().Methods(http.MethodGet).Path("/callback").Handler(Log(baseAccessLog)(http.HandlerFunc(a.callback)))
+	unauthenticatedRouter.NewRoute().Methods(http.MethodGet).Path("/callback").Handler(Log(env, audit, baseAccessLog)(http.HandlerFunc(a.callback)))
 
 	return a, nil
 }
diff --git a/pkg/portal/middleware/log.go b/pkg/portal/middleware/log.go
index 945b5f3e3..83181f614 100644
--- a/pkg/portal/middleware/log.go
+++ b/pkg/portal/middleware/log.go
@@ -5,6 +5,7 @@ package middleware
 
 import (
 	"bufio"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -12,7 +13,9 @@ import (
 
 	"github.com/sirupsen/logrus"
 
+	"github.com/Azure/ARO-RP/pkg/env"
 	utillog "github.com/Azure/ARO-RP/pkg/util/log"
+	"github.com/Azure/ARO-RP/pkg/util/log/audit"
 )
 
 type logResponseWriter struct {
@@ -50,7 +53,7 @@ func (rc *logReadCloser) Read(b []byte) (int, error) {
 	return n, err
 }
 
-func Log(baseLog *logrus.Entry) func(http.Handler) http.Handler {
+func Log(env env.Core, auditLog, baseLog *logrus.Entry) func(http.Handler) http.Handler {
 	return func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			t := time.Now()
@@ -73,16 +76,64 @@ func Log(baseLog *logrus.Entry) func(http.Handler) http.Handler {
 			})
 			log.Print("read request")
 
+			auditEntry := auditLog.WithFields(logrus.Fields{
+				audit.MetadataAdminOperation:  true,
+				audit.MetadataCreatedTime:     time.Now().UTC().Format(time.RFC3339),
+				audit.MetadataLogKind:         audit.IFXAuditLogKind,
+				audit.MetadataSource:          audit.SourceAdminPortal,
+				audit.EnvKeyAppID:             audit.SourceAdminPortal,
+				audit.EnvKeyCloudRole:         audit.CloudRoleRP,
+				audit.EnvKeyEnvironment:       env.Environment().Name,
+				audit.EnvKeyHostname:          env.Hostname(),
+				audit.EnvKeyLocation:          env.Location(),
+				audit.PayloadKeyCategory:      audit.CategoryResourceManagement,
+				audit.PayloadKeyOperationName: fmt.Sprintf("%s %s", r.Method, r.URL.Path),
+				audit.PayloadKeyCallerIdentities: []audit.CallerIdentity{
+					{
+						CallerIdentityType:  audit.CallerIdentityTypeUsername,
+						CallerIdentityValue: username,
+						CallerIPAddress:     r.RemoteAddr,
+					},
+				},
+				audit.PayloadKeyTargetResources: []audit.TargetResource{
+					{
+						TargetResourceName: r.URL.Path,
+						TargetResourceType: auditTargetResourceType(r),
+					},
+				},
+			})
+
 			defer func() {
+				statusCode := w.(*logResponseWriter).statusCode
 				log.WithFields(logrus.Fields{
 					"body_read_bytes":      r.Body.(*logReadCloser).bytes,
 					"body_written_bytes":   w.(*logResponseWriter).bytes,
 					"duration":             time.Since(t).Seconds(),
-					"response_status_code": w.(*logResponseWriter).statusCode,
+					"response_status_code": statusCode,
 				}).Print("sent response")
+
+				resultType := audit.ResultTypeSuccess
+				if statusCode >= http.StatusBadRequest {
+					resultType = audit.ResultTypeFail
+				}
+
+				auditEntry.WithFields(logrus.Fields{
+					audit.PayloadKeyResult: audit.Result{
+						ResultType:        resultType,
+						ResultDescription: fmt.Sprintf("Status code: %d", statusCode),
+					},
+				}).Info(audit.DefaultLogMessage)
 			}()
 
 			h.ServeHTTP(w, r)
 		})
 	}
 }
+
+func auditTargetResourceType(r *http.Request) string {
+	if matches := utillog.RXTolerantSubResourceID.FindStringSubmatch(r.URL.Path); matches != nil {
+		return matches[len(matches)-1]
+	}
+
+	return ""
+}
diff --git a/pkg/portal/portal.go b/pkg/portal/portal.go
index b2147faee..4d5073992 100644
--- a/pkg/portal/portal.go
+++ b/pkg/portal/portal.go
@@ -38,6 +38,7 @@ type Runnable interface {
 
 type portal struct {
 	env           env.Core
+	audit         *logrus.Entry
 	log           *logrus.Entry
 	baseAccessLog *logrus.Entry
 	l             net.Listener
@@ -67,6 +68,7 @@ type portal struct {
 }
 
 func NewPortal(env env.Core,
+	audit *logrus.Entry,
 	log *logrus.Entry,
 	baseAccessLog *logrus.Entry,
 	l net.Listener,
@@ -87,6 +89,7 @@ func NewPortal(env env.Core,
 	dialer proxy.Dialer) Runnable {
 	return &portal{
 		env:           env,
+		audit:         audit,
 		log:           log,
 		baseAccessLog: baseAccessLog,
 		l:             l,
@@ -160,14 +163,14 @@ func (p *portal) Run(ctx context.Context) error {
 	allGroups := append([]string{}, p.groupIDs...)
 	allGroups = append(allGroups, p.elevatedGroupIDs...)
 
-	p.aad, err = middleware.NewAAD(p.log, p.env, p.baseAccessLog, p.hostname, p.sessionKey, p.clientID, p.clientKey, p.clientCerts, allGroups, unauthenticatedRouter, p.verifier)
+	p.aad, err = middleware.NewAAD(p.log, p.audit, p.env, p.baseAccessLog, p.hostname, p.sessionKey, p.clientID, p.clientKey, p.clientCerts, allGroups, unauthenticatedRouter, p.verifier)
 	if err != nil {
 		return err
 	}
 
 	aadAuthenticatedRouter := r.NewRoute().Subrouter()
 	aadAuthenticatedRouter.Use(p.aad.AAD)
-	aadAuthenticatedRouter.Use(middleware.Log(p.baseAccessLog))
+	aadAuthenticatedRouter.Use(middleware.Log(p.env, p.audit, p.baseAccessLog))
 	aadAuthenticatedRouter.Use(p.aad.Redirect)
 	aadAuthenticatedRouter.Use(csrf.Protect(p.sessionKey, csrf.SameSite(csrf.SameSiteStrictMode), csrf.MaxAge(0)))
 
@@ -183,7 +186,7 @@ func (p *portal) Run(ctx context.Context) error {
 		return err
 	}
 
-	kubeconfig.New(p.log, p.baseAccessLog, p.servingCerts[0], p.elevatedGroupIDs, p.dbOpenShiftClusters, p.dbPortal, p.dialer, aadAuthenticatedRouter, unauthenticatedRouter)
+	kubeconfig.New(p.log, p.audit, p.env, p.baseAccessLog, p.servingCerts[0], p.elevatedGroupIDs, p.dbOpenShiftClusters, p.dbPortal, p.dialer, aadAuthenticatedRouter, unauthenticatedRouter)
 
 	prometheus.New(p.log, p.dbOpenShiftClusters, p.dialer, aadAuthenticatedRouter)
 
@@ -199,7 +202,7 @@ func (p *portal) Run(ctx context.Context) error {
 }
 
 func (p *portal) unauthenticatedRoutes(r *mux.Router) {
-	logger := middleware.Log(p.baseAccessLog)
+	logger := middleware.Log(p.env, p.audit, p.baseAccessLog)
 
 	r.NewRoute().Methods(http.MethodGet).Path("/healthz/ready").Handler(logger(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})))
 }
diff --git a/pkg/util/log/log.go b/pkg/util/log/log.go
index c255c59fa..2a47ee109 100644
--- a/pkg/util/log/log.go
+++ b/pkg/util/log/log.go
@@ -34,6 +34,8 @@ var (
 	RXAdminProvider = regexp.MustCompile(`^/admin/providers/([^/]+)/([^/]+)$`)
 
 	RXTolerantResourceID = regexp.MustCompile(`(?i)^(?:/admin)?/subscriptions/([^/]+)(?:/resourceGroups/([^/]+)(?:/providers/([^/]+)/([^/]+)(?:/([^/]+))?)?)?`)
+
+	RXTolerantSubResourceID = regexp.MustCompile(`(?i)^(?:/admin)?/subscriptions/([^/]+)(?:/resourceGroups/([^/]+)(?:/providers/([^/]+)/([^/]+)/([^/]+)(?:/([^/]+))?)?)?`)
 )
 
 func getBaseLogger() *logrus.Logger {
diff --git a/pkg/util/scheme/scheme.go b/pkg/util/scheme/scheme.go
index 2ec59a987..f23c6f656 100644
--- a/pkg/util/scheme/scheme.go
+++ b/pkg/util/scheme/scheme.go
@@ -4,6 +4,7 @@ package scheme
 // Licensed under the Apache License 2.0.
 
 import (
+	configv1 "github.com/openshift/api/config/v1"
 	securityv1 "github.com/openshift/api/security/v1"
 	machinev1beta1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1"
 	mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
@@ -27,6 +28,7 @@ func init() {
 	utilruntime.Must(arov1alpha1.AddToScheme(scheme.Scheme))
 	utilruntime.Must(azureproviderv1beta1.SchemeBuilder.AddToScheme(scheme.Scheme))
 	utilruntime.Must(mcv1.AddToScheme(scheme.Scheme))
+	utilruntime.Must(configv1.AddToScheme(scheme.Scheme))
 	utilruntime.Must(corev1.AddToScheme(scheme.Scheme))
 	utilruntime.Must(corev1defaults.RegisterDefaults(scheme.Scheme))
 	utilruntime.Must(appsv1.AddToScheme(scheme.Scheme))
diff --git a/vendor/github.com/openshift/library-go/LICENSE b/vendor/github.com/openshift/library-go/LICENSE
new file mode 100644
index 000000000..261eeb9e9
--- /dev/null
+++ b/vendor/github.com/openshift/library-go/LICENSE
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/vendor/github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers/status.go b/vendor/github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers/status.go
new file mode 100644
index 000000000..8491bc9e9
--- /dev/null
+++ b/vendor/github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers/status.go
@@ -0,0 +1,144 @@
+package v1helpers
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/diff"
+	"k8s.io/apimachinery/pkg/util/json"
+
+	configv1 "github.com/openshift/api/config/v1"
+)
+
+// SetStatusCondition sets the corresponding condition in conditions to newCondition.
+func SetStatusCondition(conditions *[]configv1.ClusterOperatorStatusCondition, newCondition configv1.ClusterOperatorStatusCondition) {
+	if conditions == nil {
+		conditions = &[]configv1.ClusterOperatorStatusCondition{}
+	}
+	existingCondition := FindStatusCondition(*conditions, newCondition.Type)
+	if existingCondition == nil {
+		newCondition.LastTransitionTime = metav1.NewTime(time.Now())
+		*conditions = append(*conditions, newCondition)
+		return
+	}
+
+	if existingCondition.Status != newCondition.Status {
+		existingCondition.Status = newCondition.Status
+		existingCondition.LastTransitionTime = metav1.NewTime(time.Now())
+	}
+
+	existingCondition.Reason = newCondition.Reason
+	existingCondition.Message = newCondition.Message
+}
+
+// RemoveStatusCondition removes the corresponding conditionType from conditions.
+func RemoveStatusCondition(conditions *[]configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) {
+	if conditions == nil {
+		conditions = &[]configv1.ClusterOperatorStatusCondition{}
+	}
+	newConditions := []configv1.ClusterOperatorStatusCondition{}
+	for _, condition := range *conditions {
+		if condition.Type != conditionType {
+			newConditions = append(newConditions, condition)
+		}
+	}
+
+	*conditions = newConditions
+}
+
+// FindStatusCondition finds the conditionType in conditions.
+func FindStatusCondition(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) *configv1.ClusterOperatorStatusCondition {
+	for i := range conditions {
+		if conditions[i].Type == conditionType {
+			return &conditions[i]
+		}
+	}
+
+	return nil
+}
+
+// GetStatusDiff returns a string representing change in condition status in human readable form.
+func GetStatusDiff(oldStatus configv1.ClusterOperatorStatus, newStatus configv1.ClusterOperatorStatus) string {
+	messages := []string{}
+	for _, newCondition := range newStatus.Conditions {
+		existingStatusCondition := FindStatusCondition(oldStatus.Conditions, newCondition.Type)
+		if existingStatusCondition == nil {
+			messages = append(messages, fmt.Sprintf("%s set to %s (%q)", newCondition.Type, newCondition.Status, newCondition.Message))
+			continue
+		}
+		if existingStatusCondition.Status != newCondition.Status {
+			messages = append(messages, fmt.Sprintf("%s changed from %s to %s (%q)", existingStatusCondition.Type, existingStatusCondition.Status, newCondition.Status, newCondition.Message))
+			continue
+		}
+		if existingStatusCondition.Message != newCondition.Message {
+			messages = append(messages, fmt.Sprintf("%s message changed from %q to %q", existingStatusCondition.Type, existingStatusCondition.Message, newCondition.Message))
+		}
+	}
+	for _, oldCondition := range oldStatus.Conditions {
+		// This should not happen. It means we removed old condition entirely instead of just changing its status
+		if c := FindStatusCondition(newStatus.Conditions, oldCondition.Type); c == nil {
+			messages = append(messages, fmt.Sprintf("%s was removed", oldCondition.Type))
+		}
+	}
+
+	if !equality.Semantic.DeepEqual(oldStatus.RelatedObjects, newStatus.RelatedObjects) {
+		messages = append(messages, fmt.Sprintf("status.relatedObjects changed from %q to %q", oldStatus.RelatedObjects, newStatus.RelatedObjects))
+	}
+	if !equality.Semantic.DeepEqual(oldStatus.Extension, newStatus.Extension) {
+		messages = append(messages, fmt.Sprintf("status.extension changed from %q to %q", oldStatus.Extension, newStatus.Extension))
+	}
+
+	if !equality.Semantic.DeepEqual(oldStatus.Versions, newStatus.Versions) {
+		messages = append(messages, fmt.Sprintf("status.versions changed from %q to %q", oldStatus.Versions, newStatus.Versions))
+	}
+
+	if len(messages) == 0 {
+		// ignore errors
+		originalJSON := &bytes.Buffer{}
+		json.NewEncoder(originalJSON).Encode(oldStatus)
+		newJSON := &bytes.Buffer{}
+		json.NewEncoder(newJSON).Encode(newStatus)
+		messages = append(messages, diff.StringDiff(originalJSON.String(), newJSON.String()))
+	}
+
+	return strings.Join(messages, ",")
+}
+
+// IsStatusConditionTrue returns true when the conditionType is present and set to `configv1.ConditionTrue`
+func IsStatusConditionTrue(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) bool {
+	return IsStatusConditionPresentAndEqual(conditions, conditionType, configv1.ConditionTrue)
+}
+
+// IsStatusConditionFalse returns true when the conditionType is present and set to `configv1.ConditionFalse`
+func IsStatusConditionFalse(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) bool {
+	return IsStatusConditionPresentAndEqual(conditions, conditionType, configv1.ConditionFalse)
+}
+
+// IsStatusConditionPresentAndEqual returns true when conditionType is present and equal to status.
+func IsStatusConditionPresentAndEqual(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType, status configv1.ConditionStatus) bool {
+	for _, condition := range conditions {
+		if condition.Type == conditionType {
+			return condition.Status == status
+		}
+	}
+	return false
+}
+
+// IsStatusConditionNotIn returns true when the conditionType does not match the status.
+func IsStatusConditionNotIn(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType, status ...configv1.ConditionStatus) bool {
+	for _, condition := range conditions {
+		if condition.Type == conditionType {
+			for _, s := range status {
+				if s == condition.Status {
+					return false
+				}
+			}
+			return true
+		}
+	}
+	return true
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index a8e4483ab..8f7722d4a 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -875,6 +875,9 @@ github.com/openshift/installer/pkg/types/vsphere/defaults
 github.com/openshift/installer/pkg/types/vsphere/validation
 github.com/openshift/installer/pkg/validate
 github.com/openshift/installer/pkg/version
+# github.com/openshift/library-go v0.0.0-20200909173121-1d055d971916
+## explicit
+github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers
 # github.com/openshift/machine-api-operator v0.2.1-0.20210104142355-8e6ae0acdfcf => github.com/openshift/machine-api-operator v0.2.1-0.20210212025836-cb508cd8777d
 ## explicit
 github.com/openshift/machine-api-operator/pkg/apis/machine


Test patch:
diff --git a/pkg/frontend/security_test.go b/pkg/frontend/security_test.go
index 83530ec76..d62da729e 100644
--- a/pkg/frontend/security_test.go
+++ b/pkg/frontend/security_test.go
@@ -495,6 +495,8 @@ func TestSecurity(t *testing.T) {
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
+			defer auditHook.Reset()
+
 			tlsConfig := &tls.Config{
 				RootCAs: pool,
 			}
@@ -527,7 +529,6 @@ func TestSecurity(t *testing.T) {
 			}
 
 			testlog.AssertAuditPayloads(t, auditHook, tt.wantAuditPayloads)
-			auditHook.Entries = []logrus.Entry{}
 		})
 	}
 }
diff --git a/pkg/operator/controllers/checker/serviceprincipalchecker_test.go b/pkg/operator/controllers/checker/serviceprincipalchecker_test.go
new file mode 100644
index 000000000..7bd137702
--- /dev/null
+++ b/pkg/operator/controllers/checker/serviceprincipalchecker_test.go
@@ -0,0 +1,93 @@
+package checker
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	azuretypes "github.com/openshift/installer/pkg/types/azure"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+	arofake "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned/fake"
+)
+
+// TODO - once aad.GetToken is mockable add tests for other cases
+func TestServicePrincipalValid(t *testing.T) {
+	ctx := context.Background()
+
+	for _, tt := range []struct {
+		name       string
+		aroCluster *arov1alpha1.Cluster
+		wantErr    string
+	}{
+		{
+			name:    "fail: aro cluster resource doesn't exist",
+			wantErr: `clusters.aro.openshift.io "cluster" not found`,
+		},
+		{
+			name: "fail: invalid cluster resource",
+			aroCluster: &arov1alpha1.Cluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: arov1alpha1.SingletonClusterName,
+				},
+				Spec: arov1alpha1.ClusterSpec{
+					AZEnvironment: azuretypes.PublicCloud.Name(),
+					ResourceID:    "invalid_resource",
+				},
+			},
+			wantErr: `parsing failed for invalid_resource. Invalid resource Id format`,
+		},
+		{
+			name: "fail: invalid az environment",
+			aroCluster: &arov1alpha1.Cluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: arov1alpha1.SingletonClusterName,
+				},
+				Spec: arov1alpha1.ClusterSpec{
+					AZEnvironment: "NEVERLAND",
+					ResourceID:    "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/mycluster",
+				},
+			},
+			wantErr: `autorest/azure: There is no cloud environment matching the name "NEVERLAND"`,
+		},
+		{
+			name: "fail: azure-credential secret doesn't exist",
+			aroCluster: &arov1alpha1.Cluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: arov1alpha1.SingletonClusterName,
+				},
+				Spec: arov1alpha1.ClusterSpec{
+					AZEnvironment: azuretypes.PublicCloud.Name(),
+					ResourceID:    "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myresourcegroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/mycluster",
+				},
+			},
+			wantErr: `secrets "azure-credentials" not found`,
+		},
+	} {
+		arocli := arofake.NewSimpleClientset()
+		kubernetescli := fake.NewSimpleClientset()
+
+		if tt.aroCluster != nil {
+			arocli = arofake.NewSimpleClientset(tt.aroCluster)
+		}
+
+		sp := &ServicePrincipalChecker{
+			arocli:        arocli,
+			kubernetescli: kubernetescli,
+		}
+
+		t.Run(tt.name, func(t *testing.T) {
+			err := sp.Check(ctx)
+
+			if err != nil && err.Error() != tt.wantErr ||
+				err == nil && tt.wantErr != "" {
+				t.Error(fmt.Errorf("\n%s\n !=\n%s", err.Error(), tt.wantErr))
+			}
+		})
+	}
+}
diff --git a/pkg/operator/controllers/monitoring/monitoring_controller_test.go b/pkg/operator/controllers/monitoring/monitoring_controller_test.go
index 903854e21..85c4c70a3 100644
--- a/pkg/operator/controllers/monitoring/monitoring_controller_test.go
+++ b/pkg/operator/controllers/monitoring/monitoring_controller_test.go
@@ -49,14 +49,7 @@ func TestReconcileMonitoringConfig(t *testing.T) {
 				}
 			},
 			wantConfig: `
-prometheusK8s:
-  retention: 15d
-  volumeClaimTemplate:
-    spec:
-      resources:
-        requests:
-          storage: 100Gi
-`,
+{}`,
 		},
 		{
 			name: "ConfigMap does not have data",
@@ -77,15 +70,7 @@ prometheusK8s:
 					jsonHandle: new(codec.JsonHandle),
 				}
 			},
-			wantConfig: `
-prometheusK8s:
-  retention: 15d
-  volumeClaimTemplate:
-    spec:
-      resources:
-        requests:
-          storage: 100Gi
-`,
+			wantConfig: ``,
 		},
 		{
 			name: "empty config.yaml",
@@ -109,15 +94,7 @@ prometheusK8s:
 					jsonHandle: new(codec.JsonHandle),
 				}
 			},
-			wantConfig: `
-prometheusK8s:
-  retention: 15d
-  volumeClaimTemplate:
-    spec:
-      resources:
-        requests:
-          storage: 100Gi
-`,
+			wantConfig: ``,
 		},
 		{
 			name: "settings restored to default and extra fields are preserved",
@@ -153,12 +130,8 @@ prometheusK8s:
 			},
 			wantConfig: `
 prometheusK8s:
-  retention: 15d
   volumeClaimTemplate:
     spec:
-      resources:
-        requests:
-          storage: 100Gi
       storageClassName: fast
       volumeMode: Filesystem
 `,
@@ -193,13 +166,6 @@ alertmanagerMain:
 alertmanagerMain:
   nodeSelector:
     foo: bar
-prometheusK8s:
-  retention: 15d
-  volumeClaimTemplate:
-    spec:
-      resources:
-        requests:
-          storage: 100Gi
 `,
 		},
 		{
diff --git a/pkg/operator/controllers/workaround/cleanprompvc_test.go b/pkg/operator/controllers/workaround/cleanprompvc_test.go
new file mode 100644
index 000000000..a8ccfdff4
--- /dev/null
+++ b/pkg/operator/controllers/workaround/cleanprompvc_test.go
@@ -0,0 +1,154 @@
+package workaround
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+
+	utillog "github.com/Azure/ARO-RP/pkg/util/log"
+)
+
+func TestCleanPromPVCEnsure(t *testing.T) {
+
+	pvc0 := corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "prometheus-k8s-db-prometheus-k8s-00",
+			Namespace: "openshift-monitoring",
+			Labels: map[string]string{
+				"app":        "prometheus",
+				"prometheus": "k8s",
+			},
+		},
+	}
+
+	pvc1 := corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "prometheus-k8s-db-prometheus-k8s-0",
+			Namespace: "openshift-monitoring",
+			Labels: map[string]string{
+				"app":        "prometheus",
+				"prometheus": "k8s",
+			},
+		},
+	}
+
+	pvc2 := corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "prometheus-k8s-db-prometheus-k8s-1",
+			Namespace: "openshift-monitoring",
+			Labels: map[string]string{
+				"app":        "prometheus",
+				"prometheus": "k8s",
+			},
+		},
+	}
+
+	pvc3 := corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "dummy",
+			Namespace: "openshift-monitoring",
+		},
+	}
+
+	tests := []struct {
+		name       string
+		cli        *fake.Clientset
+		wantPVCNum int
+		wantErr    error
+	}{
+		{
+			name:       "Should delete the prometheus PVCs",
+			cli:        fake.NewSimpleClientset(&pvc1, &pvc2, &pvc3),
+			wantPVCNum: 1,
+			wantErr:    nil,
+		},
+		{
+			name:       "Should not delete the prometheus PVCs, too many items",
+			cli:        fake.NewSimpleClientset(&pvc1, &pvc2, &pvc3, &pvc0),
+			wantPVCNum: 1,
+			wantErr:    nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			w := NewCleanFromPVCWorkaround(utillog.GetLogger(), tt.cli)
+			err := w.Ensure(context.Background())
+			if err != tt.wantErr {
+				t.Fatalf("Unexpected error\nwant: %v\ngot: %v", tt.wantErr, err)
+			}
+
+			pvcList, err := tt.cli.CoreV1().PersistentVolumeClaims(monitoringNamespace).List(context.Background(), metav1.ListOptions{})
+			if err != nil {
+				t.Fatalf("Unexpected error during list of PVCs: %v", err)
+			}
+			if len(pvcList.Items) != tt.wantPVCNum {
+				t.Fatalf("Unexpected number of PVCs\nwant: %d\ngot: %d", tt.wantPVCNum, len(pvcList.Items))
+			}
+		})
+	}
+}
+
+func TestCleanPromPVCIsRequired(t *testing.T) {
+	newKubernetesCli := func(config string) *fake.Clientset {
+		configMap := corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      monitoringName,
+				Namespace: monitoringNamespace,
+			},
+			Data: make(map[string]string),
+		}
+
+		configMap.Data["config.yaml"] = config
+
+		return fake.NewSimpleClientset(&configMap)
+	}
+
+	tests := []struct {
+		name         string
+		kcli         *fake.Clientset
+		wantRequired bool
+	}{
+		{
+			name: "Should not be required, persistent set true",
+			kcli: newKubernetesCli(`prometheusK8s:
+  retention: 15d
+  volumeClaimTemplate:
+    spec:
+      resources:
+        requests:
+          storage: 100Gi
+            `),
+			wantRequired: false,
+		},
+		{
+			name: "Should be required, persistent set to false",
+			kcli: newKubernetesCli(`prometheusK8s:
+  retention: ""
+  volumeClaimTemplate:
+    spec:
+      resources:
+        requests:
+          storage: ""
+            `),
+			wantRequired: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			w := NewCleanFromPVCWorkaround(utillog.GetLogger(), tt.kcli)
+			required := w.IsRequired(nil)
+			if required != tt.wantRequired {
+				t.Fatalf("Unexpected workaroud required result\nwant: %t\ngot: %t", tt.wantRequired, required)
+			}
+
+		})
+	}
+}
diff --git a/pkg/portal/kubeconfig/kubeconfig_test.go b/pkg/portal/kubeconfig/kubeconfig_test.go
index 5022d6a6e..782f291b3 100644
--- a/pkg/portal/kubeconfig/kubeconfig_test.go
+++ b/pkg/portal/kubeconfig/kubeconfig_test.go
@@ -12,15 +12,17 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/golang/mock/gomock"
 	"github.com/gorilla/mux"
-	"github.com/sirupsen/logrus"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/database/cosmosdb"
 	"github.com/Azure/ARO-RP/pkg/portal/middleware"
 	"github.com/Azure/ARO-RP/pkg/portal/util/responsewriter"
+	mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env"
 	testdatabase "github.com/Azure/ARO-RP/test/database"
+	testlog "github.com/Azure/ARO-RP/test/util/log"
 )
 
 func TestNew(t *testing.T) {
@@ -136,9 +138,17 @@ func TestNew(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			defer ctrl.Finish()
 
+			_env := mock_env.NewMockInterface(ctrl)
+			_env.EXPECT().Environment().AnyTimes().Return(&azure.PublicCloud)
+			_env.EXPECT().Hostname().AnyTimes().Return("testhost")
+			_env.EXPECT().Location().AnyTimes().Return("eastus")
+
 			aadAuthenticatedRouter := &mux.Router{}
 
-			k := New(logrus.NewEntry(logrus.StandardLogger()), nil, servingCert, elevatedGroupIDs, nil, dbPortal, nil, aadAuthenticatedRouter, &mux.Router{})
+			_, audit := testlog.NewAudit()
+			_, baseLog := testlog.New()
+			_, baseAccessLog := testlog.New()
+			k := New(baseLog, audit, _env, baseAccessLog, servingCert, elevatedGroupIDs, nil, dbPortal, nil, aadAuthenticatedRouter, &mux.Router{})
 
 			k.newToken = func() string { return password }
 
diff --git a/pkg/portal/kubeconfig/proxy_test.go b/pkg/portal/kubeconfig/proxy_test.go
index 409eb1757..b19f7c71e 100644
--- a/pkg/portal/kubeconfig/proxy_test.go
+++ b/pkg/portal/kubeconfig/proxy_test.go
@@ -15,18 +15,20 @@ import (
 	"net/http/httputil"
 	"testing"
 
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/golang/mock/gomock"
 	"github.com/gorilla/mux"
-	"github.com/sirupsen/logrus"
 	clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/database/cosmosdb"
 	"github.com/Azure/ARO-RP/pkg/portal/util/responsewriter"
+	mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env"
 	mock_proxy "github.com/Azure/ARO-RP/pkg/util/mocks/proxy"
 	utiltls "github.com/Azure/ARO-RP/pkg/util/tls"
 	testdatabase "github.com/Azure/ARO-RP/test/database"
 	"github.com/Azure/ARO-RP/test/util/listener"
+	testlog "github.com/Azure/ARO-RP/test/util/log"
 )
 
 // fakeServer returns a test listener for an HTTPS server which validates its
@@ -381,6 +383,11 @@ func TestProxy(t *testing.T) {
 			ctrl := gomock.NewController(t)
 			defer ctrl.Finish()
 
+			_env := mock_env.NewMockInterface(ctrl)
+			_env.EXPECT().Environment().AnyTimes().Return(&azure.PublicCloud)
+			_env.EXPECT().Hostname().AnyTimes().Return("testhost")
+			_env.EXPECT().Location().AnyTimes().Return("eastus")
+
 			dialer := mock_proxy.NewMockDialer(ctrl)
 			if tt.mocks != nil {
 				tt.mocks(dialer)
@@ -388,7 +395,10 @@ func TestProxy(t *testing.T) {
 
 			unauthenticatedRouter := &mux.Router{}
 
-			k := New(logrus.NewEntry(logrus.StandardLogger()), logrus.NewEntry(logrus.StandardLogger()), nil, nil, dbOpenShiftClusters, dbPortal, dialer, &mux.Router{}, unauthenticatedRouter)
+			_, audit := testlog.NewAudit()
+			_, baseLog := testlog.New()
+			_, baseAccessLog := testlog.New()
+			k := New(baseLog, audit, _env, baseAccessLog, nil, nil, dbOpenShiftClusters, dbPortal, dialer, &mux.Router{}, unauthenticatedRouter)
 
 			k.newToken = func() string { return token }
 
diff --git a/pkg/portal/middleware/aad_test.go b/pkg/portal/middleware/aad_test.go
index 740823432..44fe3ce6c 100644
--- a/pkg/portal/middleware/aad_test.go
+++ b/pkg/portal/middleware/aad_test.go
@@ -23,12 +23,12 @@ import (
 	"github.com/golang/mock/gomock"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/securecookie"
-	"github.com/sirupsen/logrus"
 	"golang.org/x/oauth2"
 
 	mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env"
 	"github.com/Azure/ARO-RP/pkg/util/roundtripper"
 	utiltls "github.com/Azure/ARO-RP/pkg/util/tls"
+	testlog "github.com/Azure/ARO-RP/test/util/log"
 )
 
 var (
@@ -81,7 +81,7 @@ func (c noopClaims) Claims(v interface{}) error {
 }
 
 func TestNewAAD(t *testing.T) {
-	_, err := NewAAD(nil, nil, nil, "", nil, "", nil, nil, nil, nil, nil)
+	_, err := NewAAD(nil, nil, nil, nil, "", nil, "", nil, nil, nil, nil, nil)
 	if err.Error() != "invalid sessionKey" {
 		t.Error(err)
 	}
@@ -162,7 +162,10 @@ func TestAAD(t *testing.T) {
 			env.EXPECT().IsDevelopmentMode().AnyTimes().Return(false)
 			env.EXPECT().TenantID().AnyTimes().Return("")
 
-			a, err := NewAAD(logrus.NewEntry(logrus.StandardLogger()), env, logrus.NewEntry(logrus.StandardLogger()), "", make([]byte, 32), "", nil, nil, nil, mux.NewRouter(), nil)
+			_, audit := testlog.NewAudit()
+			_, baseLog := testlog.New()
+			_, baseAccessLog := testlog.New()
+			a, err := NewAAD(baseLog, audit, env, baseAccessLog, "", make([]byte, 32), "", nil, nil, nil, mux.NewRouter(), nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -258,7 +261,10 @@ func TestRedirect(t *testing.T) {
 			env.EXPECT().IsDevelopmentMode().AnyTimes().Return(false)
 			env.EXPECT().TenantID().AnyTimes().Return("")
 
-			a, err := NewAAD(logrus.NewEntry(logrus.StandardLogger()), env, logrus.NewEntry(logrus.StandardLogger()), "", make([]byte, 32), "", nil, nil, nil, mux.NewRouter(), nil)
+			_, audit := testlog.NewAudit()
+			_, baseLog := testlog.New()
+			_, baseAccessLog := testlog.New()
+			a, err := NewAAD(baseLog, audit, env, baseAccessLog, "", make([]byte, 32), "", nil, nil, nil, mux.NewRouter(), nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -374,7 +380,10 @@ func TestLogout(t *testing.T) {
 			env.EXPECT().IsDevelopmentMode().AnyTimes().Return(false)
 			env.EXPECT().TenantID().AnyTimes().Return("")
 
-			a, err := NewAAD(logrus.NewEntry(logrus.StandardLogger()), env, logrus.NewEntry(logrus.StandardLogger()), "", make([]byte, 32), "", nil, nil, nil, mux.NewRouter(), nil)
+			_, audit := testlog.NewAudit()
+			_, baseLog := testlog.New()
+			_, baseAccessLog := testlog.New()
+			a, err := NewAAD(baseLog, audit, env, baseAccessLog, "", make([]byte, 32), "", nil, nil, nil, mux.NewRouter(), nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -736,7 +745,10 @@ func TestCallback(t *testing.T) {
 			env.EXPECT().IsDevelopmentMode().AnyTimes().Return(false)
 			env.EXPECT().TenantID().AnyTimes().Return("")
 
-			a, err := NewAAD(logrus.NewEntry(logrus.StandardLogger()), env, logrus.NewEntry(logrus.StandardLogger()), "", make([]byte, 32), clientID, clientkey, clientcerts, groups, mux.NewRouter(), tt.verifier)
+			_, audit := testlog.NewAudit()
+			_, baseLog := testlog.New()
+			_, baseAccessLog := testlog.New()
+			a, err := NewAAD(baseLog, audit, env, baseAccessLog, "", make([]byte, 32), clientID, clientkey, clientcerts, groups, mux.NewRouter(), tt.verifier)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -854,8 +866,10 @@ func TestClientAssertion(t *testing.T) {
 	env.EXPECT().TenantID().AnyTimes().Return("")
 
 	clientID := "00000000-0000-0000-0000-000000000000"
-
-	a, err := NewAAD(logrus.NewEntry(logrus.StandardLogger()), env, logrus.NewEntry(logrus.StandardLogger()), "", make([]byte, 32), clientID, clientkey, clientcerts, nil, mux.NewRouter(), nil)
+	_, audit := testlog.NewAudit()
+	_, baseLog := testlog.New()
+	_, baseAccessLog := testlog.New()
+	a, err := NewAAD(baseLog, audit, env, baseAccessLog, "", make([]byte, 32), clientID, clientkey, clientcerts, nil, mux.NewRouter(), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/pkg/portal/middleware/log_test.go b/pkg/portal/middleware/log_test.go
index 1b8d2af83..153ace4e7 100644
--- a/pkg/portal/middleware/log_test.go
+++ b/pkg/portal/middleware/log_test.go
@@ -9,18 +9,31 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"strings"
 	"testing"
 
+	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/golang/mock/gomock"
 	"github.com/onsi/gomega"
 	"github.com/onsi/gomega/types"
 	"github.com/sirupsen/logrus"
 
+	"github.com/Azure/ARO-RP/pkg/util/log/audit"
+	mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env"
 	testlog "github.com/Azure/ARO-RP/test/util/log"
 )
 
 func TestLog(t *testing.T) {
 	h, log := testlog.New()
+	ah, auditLog := testlog.NewAudit()
+
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+	_env := mock_env.NewMockInterface(controller)
+	_env.EXPECT().Environment().AnyTimes().Return(&azure.PublicCloud)
+	_env.EXPECT().Hostname().AnyTimes().Return("testhost")
+	_env.EXPECT().Location().AnyTimes().Return("eastus")
 
 	ctx := context.WithValue(context.Background(), ContextKeyUsername, "username")
 	r, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://localhost/", strings.NewReader("body"))
@@ -32,7 +45,8 @@ func TestLog(t *testing.T) {
 
 	w := httptest.NewRecorder()
 
-	Log(log)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	// chain a custom handler with the Log middleware to mutate the request
+	Log(_env, auditLog, log)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		r.URL = nil // mutate the request
 
 		_ = w.(http.Hijacker) // must implement http.Hijacker
@@ -76,7 +90,115 @@ func TestLog(t *testing.T) {
 		t.Error(err)
 	}
 
+	// only one audit log is associated with the one POST request
+	expectedAudit := []*audit.Payload{
+		{
+			EnvVer:               audit.IFXAuditVersion,
+			EnvName:              audit.IFXAuditName,
+			EnvFlags:             257,
+			EnvAppID:             audit.SourceAdminPortal,
+			EnvCloudName:         _env.Environment().Name,
+			EnvCloudRole:         audit.CloudRoleRP,
+			EnvCloudRoleInstance: _env.Hostname(),
+			EnvCloudEnvironment:  _env.Environment().Name,
+			EnvCloudLocation:     _env.Location(),
+			EnvCloudVer:          audit.IFXAuditCloudVer,
+			CallerIdentities: []audit.CallerIdentity{
+				{
+					CallerIdentityType:  audit.CallerIdentityTypeUsername,
+					CallerIdentityValue: "username",
+					CallerIPAddress:     "127.0.0.1:1234",
+				},
+			},
+			Category:      "ResourceManagement",
+			OperationName: "POST /",
+			Result: audit.Result{
+				ResultType:        "Success",
+				ResultDescription: "Status code: 200",
+			},
+			TargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "",
+					TargetResourceName: "/",
+				},
+			},
+		},
+	}
+	testlog.AssertAuditPayloads(t, ah, expectedAudit)
+
 	for _, e := range h.Entries {
 		fmt.Println(e)
 	}
+
+}
+
+func TestAuditTargetResourceData(t *testing.T) {
+	var (
+		testSubscription  = "test-sub"
+		testResourceGroup = "test-rg"
+		testResource      = "test-resource"
+	)
+
+	var testCases = []struct {
+		url          string
+		expectedType string
+		expectedName string
+	}{
+		{
+			url:          "/",
+			expectedName: "/",
+		},
+		{
+			url:          "/index.html",
+			expectedName: "/index.html",
+		},
+		{
+			url:          "/lib/bootstrap-4.5.2.min.css",
+			expectedName: "/lib/bootstrap-4.5.2.min.css",
+		},
+		{
+			url:          "/api/clusters",
+			expectedName: "/api/clusters",
+		},
+		{
+			url:          "/api/logout",
+			expectedName: "/api/logout",
+		},
+		{
+			url:          fmt.Sprintf("/subscriptions/%s/resourcegroups/%s/providers/microsoft.redhatopenshift/openshiftclusters/%s/ssh/new", testSubscription, testResourceGroup, testResource),
+			expectedName: fmt.Sprintf("/subscriptions/%s/resourcegroups/%s/providers/microsoft.redhatopenshift/openshiftclusters/%s/ssh/new", testSubscription, testResourceGroup, testResource),
+			expectedType: "ssh",
+		},
+		{
+			url:          fmt.Sprintf("/subscriptions/%s/resourcegroups/%s/providers/microsoft.redhatopenshift/openshiftclusters/%s/kubeconfig/new", testSubscription, testResourceGroup, testResource),
+			expectedName: fmt.Sprintf("/subscriptions/%s/resourcegroups/%s/providers/microsoft.redhatopenshift/openshiftclusters/%s/kubeconfig/new", testSubscription, testResourceGroup, testResource),
+			expectedType: "kubeconfig",
+		},
+		{
+			url:          fmt.Sprintf("/subscriptions/%s/resourcegroups/%s/providers/microsoft.redhatopenshift/openshiftclusters/%s/kubeconfig/proxy", testSubscription, testResourceGroup, testResource),
+			expectedName: fmt.Sprintf("/subscriptions/%s/resourcegroups/%s/providers/microsoft.redhatopenshift/openshiftclusters/%s/kubeconfig/proxy", testSubscription, testResourceGroup, testResource),
+			expectedType: "kubeconfig",
+		},
+		{
+			url:          fmt.Sprintf("/subscriptions/%s/resourcegroups/%s/providers/microsoft.redhatopenshift/openshiftclusters/%s/prometheus", testSubscription, testResourceGroup, testResource),
+			expectedName: fmt.Sprintf("/subscriptions/%s/resourcegroups/%s/providers/microsoft.redhatopenshift/openshiftclusters/%s/prometheus", testSubscription, testResourceGroup, testResource),
+			expectedType: "prometheus",
+		},
+	}
+
+	for _, tc := range testCases {
+		parsedURL, err := url.Parse(tc.url)
+		if err != nil {
+			t.Fatal("unexpected error: ", err)
+		}
+
+		r := &http.Request{URL: parsedURL}
+		if actual := auditTargetResourceType(r); tc.expectedType != actual {
+			t.Errorf("%s: expected: %s, actual: %s", tc.url, tc.expectedType, actual)
+		}
+
+		if actual := r.URL.Path; tc.expectedName != actual {
+			t.Errorf("%s: expected: %s, actual: %s", tc.url, tc.expectedName, actual)
+		}
+	}
 }
diff --git a/pkg/portal/security_test.go b/pkg/portal/security_test.go
index b0e704b09..ebc3d7ab1 100644
--- a/pkg/portal/security_test.go
+++ b/pkg/portal/security_test.go
@@ -9,21 +9,25 @@ import (
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/golang/mock/gomock"
 	"github.com/gorilla/securecookie"
 	"github.com/gorilla/sessions"
 	"github.com/sirupsen/logrus"
 
 	"github.com/Azure/ARO-RP/pkg/portal/middleware"
+	"github.com/Azure/ARO-RP/pkg/util/log/audit"
 	mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env"
 	utiltls "github.com/Azure/ARO-RP/pkg/util/tls"
 	testdatabase "github.com/Azure/ARO-RP/test/database"
 	"github.com/Azure/ARO-RP/test/util/listener"
+	testlog "github.com/Azure/ARO-RP/test/util/log"
 )
 
 var (
@@ -34,6 +38,10 @@ func TestSecurity(t *testing.T) {
 	ctx := context.Background()
 	log := logrus.NewEntry(logrus.StandardLogger())
 
+	_, portalAccessLog := testlog.New()
+	_, portalLog := testlog.New()
+	auditHook, portalAuditLog := testlog.NewAudit()
+
 	controller := gomock.NewController(t)
 	defer controller.Finish()
 
@@ -41,6 +49,8 @@ func TestSecurity(t *testing.T) {
 	_env.EXPECT().IsDevelopmentMode().AnyTimes().Return(false)
 	_env.EXPECT().Location().AnyTimes().Return("eastus")
 	_env.EXPECT().TenantID().AnyTimes().Return("00000000-0000-0000-0000-000000000001")
+	_env.EXPECT().Environment().AnyTimes().Return(&azure.PublicCloud)
+	_env.EXPECT().Hostname().AnyTimes().Return("testhost")
 
 	l := listener.NewListener()
 	defer l.Close()
@@ -76,7 +86,7 @@ func TestSecurity(t *testing.T) {
 		},
 	}
 
-	p := NewPortal(_env, log, log, l, sshl, nil, "", serverkey, servercerts, "", nil, nil, make([]byte, 32), sshkey, nil, elevatedGroupIDs, dbOpenShiftClusters, dbPortal, nil)
+	p := NewPortal(_env, portalAuditLog, portalLog, portalAccessLog, l, sshl, nil, "", serverkey, servercerts, "", nil, nil, make([]byte, 32), sshkey, nil, elevatedGroupIDs, dbOpenShiftClusters, dbPortal, nil)
 	go func() {
 		err := p.Run(ctx)
 		if err != nil {
@@ -90,24 +100,47 @@ func TestSecurity(t *testing.T) {
 		checkResponse                 func(*testing.T, bool, bool, *http.Response)
 		unauthenticatedWantStatusCode int
 		authenticatedWantStatusCode   int
+		wantAuditOperation            string
+		wantAuditTargetResources      []audit.TargetResource
 	}{
 		{
 			name: "/",
 			request: func() (*http.Request, error) {
 				return http.NewRequest(http.MethodGet, "https://server/", nil)
 			},
+			wantAuditOperation: "GET /",
+			wantAuditTargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "",
+					TargetResourceName: "/",
+				},
+			},
 		},
 		{
 			name: "/index.js",
 			request: func() (*http.Request, error) {
 				return http.NewRequest(http.MethodGet, "https://server/index.js", nil)
 			},
+			wantAuditOperation: "GET /index.js",
+			wantAuditTargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "",
+					TargetResourceName: "/index.js",
+				},
+			},
 		},
 		{
 			name: "/api/clusters",
 			request: func() (*http.Request, error) {
 				return http.NewRequest(http.MethodGet, "https://server/api/clusters", nil)
 			},
+			wantAuditOperation: "GET /api/clusters",
+			wantAuditTargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "",
+					TargetResourceName: "/api/clusters",
+				},
+			},
 		},
 		{
 			name: "/api/logout",
@@ -115,6 +148,13 @@ func TestSecurity(t *testing.T) {
 				return http.NewRequest(http.MethodPost, "https://server/api/logout", nil)
 			},
 			authenticatedWantStatusCode: http.StatusSeeOther,
+			wantAuditOperation:          "POST /api/logout",
+			wantAuditTargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "",
+					TargetResourceName: "/api/logout",
+				},
+			},
 		},
 		{
 			name: "/callback",
@@ -122,6 +162,13 @@ func TestSecurity(t *testing.T) {
 				return http.NewRequest(http.MethodGet, "https://server/callback", nil)
 			},
 			authenticatedWantStatusCode: http.StatusTemporaryRedirect,
+			wantAuditOperation:          "GET /callback",
+			wantAuditTargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "",
+					TargetResourceName: "/callback",
+				},
+			},
 		},
 		{
 			name: "/healthz/ready",
@@ -129,12 +176,26 @@ func TestSecurity(t *testing.T) {
 				return http.NewRequest(http.MethodGet, "https://server/healthz/ready", nil)
 			},
 			unauthenticatedWantStatusCode: http.StatusOK,
+			wantAuditOperation:            "GET /healthz/ready",
+			wantAuditTargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "",
+					TargetResourceName: "/healthz/ready",
+				},
+			},
 		},
 		{
 			name: "/kubeconfig/new",
 			request: func() (*http.Request, error) {
 				return http.NewRequest(http.MethodPost, "https://server/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroupName/providers/microsoft.redhatopenshift/openshiftclusters/resourceName/kubeconfig/new", nil)
 			},
+			wantAuditOperation: "POST /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourcegroupname/providers/microsoft.redhatopenshift/openshiftclusters/resourcename/kubeconfig/new",
+			wantAuditTargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "kubeconfig",
+					TargetResourceName: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourcegroupname/providers/microsoft.redhatopenshift/openshiftclusters/resourcename/kubeconfig/new",
+				},
+			},
 		},
 		{
 			name: "/prometheus",
@@ -142,6 +203,13 @@ func TestSecurity(t *testing.T) {
 				return http.NewRequest(http.MethodPost, "https://server/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroupName/providers/microsoft.redhatopenshift/openshiftclusters/resourceName/prometheus", nil)
 			},
 			authenticatedWantStatusCode: http.StatusTemporaryRedirect,
+			wantAuditOperation:          "POST /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourcegroupname/providers/microsoft.redhatopenshift/openshiftclusters/resourcename/prometheus",
+			wantAuditTargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "prometheus",
+					TargetResourceName: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourcegroupname/providers/microsoft.redhatopenshift/openshiftclusters/resourcename/prometheus",
+				},
+			},
 		},
 		{
 			name: "/ssh/new",
@@ -168,6 +236,13 @@ func TestSecurity(t *testing.T) {
 					}
 				}
 			},
+			wantAuditOperation: "POST /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourcegroupname/providers/microsoft.redhatopenshift/openshiftclusters/resourcename/ssh/new",
+			wantAuditTargetResources: []audit.TargetResource{
+				{
+					TargetResourceType: "ssh",
+					TargetResourceName: "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourcegroupname/providers/microsoft.redhatopenshift/openshiftclusters/resourcename/ssh/new",
+				},
+			},
 		},
 		{
 			name: "/doesnotexist",
@@ -201,6 +276,8 @@ func TestSecurity(t *testing.T) {
 			},
 		} {
 			t.Run(tt2.name+tt.name, func(t *testing.T) {
+				defer auditHook.Reset()
+
 				req, err := tt.request()
 				if err != nil {
 					t.Fatal(err)
@@ -243,6 +320,36 @@ func TestSecurity(t *testing.T) {
 				if tt.checkResponse != nil {
 					tt.checkResponse(t, tt2.authenticated, tt2.elevated, resp)
 				}
+
+				// no audit logs for https://server/doesnotexist
+				if tt.authenticatedWantStatusCode == http.StatusNotFound {
+					return
+				}
+
+				// skipping https://server because the http.ServeContent() calls in the
+				// portal's serve() and index() handlers[1] issued a call to io.Copy()[2]
+				// causes a race condition with the audit hook. The response was returned
+				// to the client and the testlog.AssertAuditPayloads() was called immediately,
+				// while the audit hook was still in-flight.
+				//
+				// note that the audit logs will still be recorded and emitted by the audit
+				// hook, so this is a non-issue in the Geneva environment.
+				//
+				// [1] https://github.com/Azure/ARO-RP/blob/master/pkg/portal/portal.go#L222-L247
+				// [2] https://go.googlesource.com/go/+/go1.16.2/src/net/http/fs.go#337
+				if tt.name == "/" || tt.name == "/index.js" {
+					return
+				}
+
+				payload := auditPayloadFixture()
+				payload.OperationName = tt.wantAuditOperation
+				payload.TargetResources = tt.wantAuditTargetResources
+				payload.Result.ResultDescription = fmt.Sprintf("Status code: %d", tt2.wantStatusCode)
+				if tt2.authenticated && tt.name != "/callback" && tt.name != "/healthz/ready" {
+					payload.CallerIdentities[0].CallerIdentityValue = "username"
+				}
+
+				testlog.AssertAuditPayloads(t, auditHook, []*audit.Payload{payload})
 			})
 		}
 	}
@@ -282,3 +389,29 @@ func addAuth(req *http.Request, groups []string) error {
 
 	return nil
 }
+
+func auditPayloadFixture() *audit.Payload {
+	return &audit.Payload{
+		EnvVer:               audit.IFXAuditVersion,
+		EnvName:              audit.IFXAuditName,
+		EnvFlags:             257,
+		EnvAppID:             audit.SourceAdminPortal,
+		EnvCloudName:         azure.PublicCloud.Name,
+		EnvCloudRole:         audit.CloudRoleRP,
+		EnvCloudRoleInstance: "testhost",
+		EnvCloudEnvironment:  azure.PublicCloud.Name,
+		EnvCloudLocation:     "eastus",
+		EnvCloudVer:          1,
+		CallerIdentities: []audit.CallerIdentity{
+			{
+				CallerDisplayName:  "",
+				CallerIdentityType: audit.CallerIdentityTypeUsername,
+				CallerIPAddress:    "bufferedpipe",
+			},
+		},
+		Category: audit.CategoryResourceManagement,
+		Result: audit.Result{
+			ResultType: audit.ResultTypeSuccess,
+		},
+	}
+}