Repository: Azure/ARO-RP
Pull request #2633 (2023-01-12 21:49:23)

Onebranch template-buildrp-buildaro: fix logic
### Which issue this PR addresses:

<!--
Please include a link to the ADO work item as well as any GitHub issues.

Usage: `Fixes #<GitHub issue number>`, or `Fixes (paste link of issue)`.
-->
Fixes logic causing the pullrequest pipeline to fail



Patch:
diff --git a/.dockerignore b/.dockerignore
index fd6f0c66b..9171a0166 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -3,7 +3,6 @@ gomock_reflect_*
 /.vscode
 /*.kubeconfig
 /dev-config.yaml
-/env*
 /id_rsa
 /pyenv*
 /mdm_statsd.socket
diff --git a/.pipelines/onebranch/templates/template-buildrp-buildaro.yml b/.pipelines/onebranch/templates/template-buildrp-buildaro.yml
index d00b2320f..b2a952e64 100644
--- a/.pipelines/onebranch/templates/template-buildrp-buildaro.yml
+++ b/.pipelines/onebranch/templates/template-buildrp-buildaro.yml
@@ -9,10 +9,10 @@ steps:
       export COMMIT=$(git rev-parse --short=7 HEAD)$([[ $(git status --porcelain) = "" ]] || echo -dirty)
       if [ -z "$TAG" ];
       then
-        if [ -z "$is_official_release"]
+        if [ "$is_official_release" = "True" ]
         then
           git describe --exact-match
-          echo "Ensure there is an annotated tag (git tag -a) for git commit $(COMMIT)"
+          echo "Ensure there is an annotated tag (git tag -a) for git commit ${COMMIT}"
           exit 1
         fi
         export VERSION=${COMMIT}
diff --git a/hack/devtools/deploy-shared-env.sh b/hack/devtools/deploy-shared-env.sh
index 4705cbccd..990eecec4 100644
--- a/hack/devtools/deploy-shared-env.sh
+++ b/hack/devtools/deploy-shared-env.sh
@@ -76,8 +76,8 @@ deploy_aks_dev() {
         -n aks-development \
         --template-file pkg/deploy/assets/aks-development.json \
         --parameters \
-            "adminObjectId=$ADMIN_OBJECT_ID" \
             "dnsZone=$DOMAIN_NAME" \
+            "keyvaultPrefix=$KEYVAULT_PREFIX" \
             "sshRSAPublicKey=$(<secrets/proxy_id_rsa.pub)" >/dev/null
 }
 
diff --git a/pkg/deploy/assets/aks-development.json b/pkg/deploy/assets/aks-development.json
index 3e6272e39..59268f07f 100644
--- a/pkg/deploy/assets/aks-development.json
+++ b/pkg/deploy/assets/aks-development.json
@@ -2,12 +2,6 @@
     "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
     "contentVersion": "1.0.0.0",
     "parameters": {
-        "adminObjectId": {
-            "type": "string",
-            "metadata": {
-                "description": "The AAD admin group's object ID."
-            }
-        },
         "dnsZone": {
             "type": "string",
             "metadata": {
@@ -33,33 +27,22 @@
                 "description": "Optional DNS prefix to use with hosted Kubernetes API server FQDN."
             }
         },
-        "availabilityZones": {
-            "defaultValue": [
-                "1",
-                "2",
-                "3"
-            ],
-            "type": "array",
-            "metadata": {
-                "description": "Specifies the availability zones for the system node pool."
-            }
-        },
         "agentCount": {
-            "defaultValue": 3,
+            "defaultValue": 2,
             "type": "int",
             "metadata": {
                 "description": "The number of agent nodes for the cluster."
             }
         },
         "agentMinCount": {
-            "defaultValue": 3,
+            "defaultValue": 2,
             "type": "int",
             "metadata": {
                 "description": "The min number of agent nodes."
             }
         },
         "agentMaxCount": {
-            "defaultValue": 10,
+            "defaultValue": 3,
             "type": "int",
             "metadata": {
                 "description": "The max number of agent nodes."
@@ -102,7 +85,7 @@
             }
         },
         "kubernetesVersion": {
-            "defaultValue": "1.23.5",
+            "defaultValue": "1.23.12",
             "type": "string",
             "metadata": {
                 "description": "The version of Kubernetes."
@@ -135,7 +118,7 @@
         },
         "vnetAddressPrefix": {
             "type": "string",
-            "defaultValue": "10.128.0.0/9",
+            "defaultValue": "10.128.0.0/14",
             "metadata": {
                 "description": "VNET address prefix"
             }
@@ -149,7 +132,7 @@
         },
         "subnetPrefix": {
             "type": "string",
-            "defaultValue": "10.128.64.0/18",
+            "defaultValue": "10.128.8.0/21",
             "metadata": {
                 "description": "Subnet address prefix"
             }
@@ -162,7 +145,7 @@
             }
         },
         "podSubnetPrefix": {
-            "defaultValue": "10.129.0.0/16",
+            "defaultValue": "10.128.64.0/18",
             "type": "string",
             "metadata": {
                 "description": "Specifies the address prefix of the subnet hosting the pods of the AKS cluster."
@@ -251,6 +234,17 @@
             "metadata": {
                 "description": "Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster."
             }
+        },
+        "Ev2ShellSubnetName": {
+            "type": "string",
+            "defaultValue": "EV2-Shell-Containers",
+            "metadata": {
+                "description": "The Subnet that is used to run EV2 deploy containers"
+            }
+        },
+        "Ev2SubnetPrefix": {
+            "type": "string",
+            "defaultValue": "10.128.0.0/25"
         }
     },
     "variables": {
@@ -260,6 +254,11 @@
         "vnetId": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]",
         "vnetSubnetId": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), parameters('vnetSubnetName'))]",
         "podSubnetId": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), parameters('podSubnetName'))]",
+        "aksClusterAdminRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions/', '0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8')]",
+        "rpServicePrincipalId": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', concat('aro-rp-', resourceGroup().location))]",
+        "aksClusterAdminRoleAssignmentName": "[guid(resourceGroup().location, parameters('aksClusterName'), variables('aksClusterAdminRoleId'), variables('rpServicePrincipalId'))]",
+        "aksClusterRbacClusterAdminRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions/', 'b1ff04bb-8a4e-4dc4-8eb5-8693973ce19b')]",
+        "aksClusterRbacClusterAdminRoleAssignmentName": "[guid(resourceGroup().location, parameters('aksClusterName'), variables('aksClusterRbacClusterAdminRoleId'), variables('rpServicePrincipalId'))]",
         "networkContributorRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions/', '4d97b98b-1d4f-4787-a291-c67834d212e7')]",
         "aksClusterUserDefinedManagedIdentityName": "[concat(parameters('aksClusterName'), '-msi')]",
         "aksClusterUserDefinedManagedIdentityId": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('aksClusterUserDefinedManagedIdentityName'))]",
@@ -280,6 +279,47 @@
             "location": "[resourceGroup().location]",
             "apiVersion": "2021-04-01"
         },
+        {
+            "type": "Microsoft.Network/networkSecurityGroups",
+            "apiVersion": "2021-08-01",
+            "location": "[resourceGroup().location]",
+            "name": "aks-ev2-shellext-nsg",
+            "properties": {}
+        },
+        {
+            "type": "Microsoft.Network/publicIPAddresses",
+            "apiVersion": "2021-02-01",
+            "name": "aks-ev2-shellext-nat-gateway-pip",
+            "location": "[resourceGroup().location]",
+            "sku": {
+                "name": "Standard"
+            },
+            "properties": {
+                "publicIPAddressVersion": "IPv4",
+                "publicIPAllocationMethod": "Static",
+                "idleTimeoutInMinutes": 4
+            }
+        },
+        {
+            "type": "Microsoft.Network/natGateways",
+            "apiVersion": "2021-02-01",
+            "name": "aks-ev2-shellext-nat-gateway",
+            "location": "[resourceGroup().location]",
+            "sku": {
+                "name": "Standard"
+            },
+            "dependsOn": [
+                "[resourceId('Microsoft.Network/publicIPAddresses', 'aks-ev2-shellext-nat-gateway-pip')]"
+            ],
+            "properties": {
+                "idleTimeoutInMinutes": 4,
+                "publicIpAddresses": [
+                    {
+                        "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'aks-ev2-shellext-nat-gateway-pip')]"
+                    }
+                ]
+            }
+        },
         {
             "type": "Microsoft.Network/virtualNetworks",
             "apiVersion": "2022-01-01",
@@ -287,7 +327,10 @@
             "name": "[parameters('vnetName')]",
             "dependsOn": [
                 "[variables('aksNsgId')]",
-                "[variables('aksPodNsgId')]"
+                "[variables('aksPodNsgId')]",
+                "[resourceId('Microsoft.Network/networkSecurityGroups', 'aks-ev2-shellext-nsg')]",
+                "[resourceId('Microsoft.Network/publicIPAddresses', 'aks-ev2-shellext-nat-gateway-pip')]",
+                "[resourceId('Microsoft.Network/natGateways', 'aks-ev2-shellext-nat-gateway')]"
             ],
             "properties": {
                 "addressSpace": {
@@ -326,6 +369,11 @@
                             "networkSecurityGroup": {
                                 "id": "[variables('aksPodNsgId')]"
                             },
+                            "serviceEndpoints": [
+                                {
+                                    "service": "Microsoft.Storage"
+                                }
+                            ],
                             "delegations": [
                                 {
                                     "name": "AKS",
@@ -335,6 +383,28 @@
                                 }
                             ]
                         }
+                    },
+                    {
+                        "name": "[parameters('Ev2ShellSubnetName')]",
+                        "properties": {
+                            "addressPrefix": "[parameters('Ev2SubnetPrefix')]",
+                            "networkSecurityGroup": {
+                                "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'aks-ev2-shellext-nsg')]"
+                            },
+                            "natGateway": {
+                                "id": "[resourceId('Microsoft.Network/natGateways', 'aks-ev2-shellext-nat-gateway')]"
+                            },
+                            "delegations": [
+                                {
+                                    "name": "Microsoft.ContainerInstance.containerGroups",
+                                    "properties": {
+                                        "serviceName": "Microsoft.ContainerInstance/containerGroups"
+                                    }
+                                }
+                            ],
+                            "privateEndpointNetworkPolicies": "Disabled",
+                            "privateLinkServiceNetworkPolicies": "Enabled"
+                        }
                     }
                 ]
             }
@@ -523,8 +593,7 @@
                         "vmSize": "[parameters('agentVMSize')]",
                         "vnetSubnetID": "[variables('vnetSubnetID')]",
                         "podSubnetID": "[variables('podSubnetId')]",
-                        "maxPods": "[parameters('maxPods')]",
-                        "availabilityZones": "[parameters('availabilityZones')]"
+                        "maxPods": "[parameters('maxPods')]"
                     }
                 ],
                 "networkProfile": {
@@ -546,42 +615,52 @@
                     "scale-down-unready-time": "[parameters('autoScalerProfileScaleDownUnreadyTime')]",
                     "scale-down-utilization-threshold": "[parameters('autoScalerProfileUtilizationThreshold')]",
                     "max-graceful-termination-sec": "[parameters('autoScalerProfileMaxGracefulTerminationSec')]"
+                },
+                "autoUpgradeProfile": {
+                    "upgradeChannel": "node-image"
                 }
             }
         },
         {
-            "type": "Microsoft.Network/dnszones/CNAME",
-            "apiVersion": "2018-05-01",
-            "name": "[concat(parameters('dnsZone'), '/api.aks1')]",
+            "type": "Microsoft.Authorization/roleAssignments",
+            "apiVersion": "2022-04-01",
+            "name": "[variables('aksClusterRbacClusterAdminRoleAssignmentName')]",
+            "scope": "[variables('aksClusterId')]",
             "dependsOn": [
-                "[parameters('aksClusterName')]"
+                "[variables('aksClusterId')]"
             ],
             "properties": {
-                "TTL": 300,
-                "CNAMERecord": {
-                    "cname": "[reference(variables('aksClusterId')).fqdn]"
-                },
-                "targetResource": {}
+                "roleDefinitionId": "[variables('aksClusterRbacClusterAdminRoleId')]",
+                "principalId": "[reference(variables('rpServicePrincipalId'), '2018-11-30').principalId]",
+                "principalType": "ServicePrincipal"
             }
         },
         {
-            "apiVersion": "2019-09-01",
-            "type": "Microsoft.KeyVault/vaults",
-            "name": "[concat(parameters('keyvaultPrefix'), '-aks')]",
-            "location": "[resourceGroup().location]",
+            "type": "Microsoft.Authorization/roleAssignments",
+            "apiVersion": "2022-04-01",
+            "name": "[variables('aksClusterAdminRoleAssignmentName')]",
+            "scope": "[variables('aksClusterId')]",
+            "dependsOn": [
+                "[variables('aksClusterId')]"
+            ],
+            "properties": {
+                "roleDefinitionId": "[variables('aksClusterAdminRoleId')]",
+                "principalId": "[reference(variables('rpServicePrincipalId'), '2018-11-30').principalId]",
+                "principalType": "ServicePrincipal"
+            }
+        },
+        {
+            "name": "[concat(parameters('keyvaultPrefix'), '-svc/add')]",
+            "type": "Microsoft.KeyVault/vaults/accessPolicies",
+            "apiVersion": "2021-10-01",
             "dependsOn": [
                 "[variables('aksClusterId')]"
             ],
             "properties": {
-                "tenantId": "[subscription().tenantId]",
-                "sku": {
-                    "family": "A",
-                    "name": "standard"
-                },
                 "accessPolicies": [
                     {
                         "tenantId": "[subscription().tenantId]",
-                        "objectId": "[reference(variables('aksClusterId'), '2020-12-01', 'Full').properties.addonProfiles.azureKeyvaultSecretsProvider.identity.objectId]",
+                        "objectId": "[reference(variables('aksClusterId'), '2020-12-01', 'Full').properties.identityProfile.kubeletidentity.objectId]",
                         "permissions": {
                             "secrets": [
                                 "get",
@@ -591,26 +670,8 @@
                                 "get"
                             ]
                         }
-                    },
-                    {
-                        "tenantId": "[subscription().tenantId]",
-                        "objectId": "[parameters('adminObjectId')]",
-                        "permissions": {
-                            "secrets": [
-                                "get",
-                                "set",
-                                "list"
-                            ],
-                            "certificates": [
-                                "delete",
-                                "get",
-                                "import",
-                                "list"
-                            ]
-                        }
                     }
-                ],
-                "enableSoftDelete": true
+                ]
             }
         }
     ]
diff --git a/pkg/frontend/admin_openshiftcluster_resources_list.go b/pkg/frontend/admin_openshiftcluster_resources_list.go
index a3b0490be..9b7825695 100644
--- a/pkg/frontend/admin_openshiftcluster_resources_list.go
+++ b/pkg/frontend/admin_openshiftcluster_resources_list.go
@@ -5,6 +5,7 @@ package frontend
 
 import (
 	"context"
+	"io"
 	"net/http"
 	"path/filepath"
 	"strings"
@@ -14,6 +15,7 @@ import (
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/database/cosmosdb"
+	"github.com/Azure/ARO-RP/pkg/frontend/adminactions"
 	"github.com/Azure/ARO-RP/pkg/frontend/middleware"
 )
 
@@ -22,13 +24,22 @@ func (f *frontend) listAdminOpenShiftClusterResources(w http.ResponseWriter, r *
 	log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry)
 	r.URL.Path = filepath.Dir(r.URL.Path)
 
-	b, err := f._listAdminOpenShiftClusterResources(ctx, r, log)
+	reader, writer := io.Pipe()
+	err := f._listAdminOpenShiftClusterResources(ctx, r, writer, log)
 
-	adminReply(log, w, nil, b, err)
+	f.streamResponder.AdminReplyStream(log, w, nil, reader, err)
 }
 
 func (f *frontend) _listAdminOpenShiftClusterResources(
-	ctx context.Context, r *http.Request, log *logrus.Entry) ([]byte, error) {
+	ctx context.Context, r *http.Request, writer io.WriteCloser, log *logrus.Entry) error {
+	a, err := f.newStreamAzureAction(ctx, r, log)
+	if err != nil {
+		return err
+	}
+	return a.WriteToStream(ctx, writer)
+}
+
+func (f *frontend) newStreamAzureAction(ctx context.Context, r *http.Request, log *logrus.Entry) (adminactions.AzureActions, error) {
 	vars := mux.Vars(r)
 	resourceID := strings.TrimPrefix(r.URL.Path, "/admin")
 
@@ -52,5 +63,5 @@ func (f *frontend) _listAdminOpenShiftClusterResources(
 		return nil, err
 	}
 
-	return a.ResourcesList(ctx)
+	return a, err
 }
diff --git a/pkg/frontend/adminactions/azureactions.go b/pkg/frontend/adminactions/azureactions.go
index 9febb2b67..45a4ce154 100644
--- a/pkg/frontend/adminactions/azureactions.go
+++ b/pkg/frontend/adminactions/azureactions.go
@@ -7,9 +7,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 
 	mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
+	mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
 	"github.com/sirupsen/logrus"
 
 	"github.com/Azure/ARO-RP/pkg/api"
@@ -24,7 +26,9 @@ import (
 
 // AzureActions contains those actions which rely solely on Azure clients, not using any k8s clients
 type AzureActions interface {
-	ResourcesList(ctx context.Context) ([]byte, error)
+	GroupResourceList(ctx context.Context) ([]mgmtfeatures.GenericResourceExpanded, error)
+	ResourcesList(ctx context.Context, resources []mgmtfeatures.GenericResourceExpanded, writer io.WriteCloser) error
+	WriteToStream(ctx context.Context, writer io.WriteCloser) error
 	NICReconcileFailedState(ctx context.Context, nicName string) error
 	VMRedeployAndWait(ctx context.Context, vmName string) error
 	VMStartAndWait(ctx context.Context, vmName string) error
diff --git a/pkg/frontend/adminactions/resources_list.go b/pkg/frontend/adminactions/resources_list.go
index c95856101..d05658c7d 100644
--- a/pkg/frontend/adminactions/resources_list.go
+++ b/pkg/frontend/adminactions/resources_list.go
@@ -5,9 +5,10 @@ package adminactions
 
 import (
 	"context"
-	"encoding/json"
+	"io"
 
 	mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
+	mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
 	"github.com/Azure/go-autorest/autorest/azure"
 
 	"github.com/Azure/ARO-RP/pkg/util/arm"
@@ -16,17 +17,22 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
 
-func (a *azureActions) ResourcesList(ctx context.Context) ([]byte, error) {
+var commaAsByteArray = []byte(",")
+
+func (a *azureActions) GroupResourceList(ctx context.Context) ([]mgmtfeatures.GenericResourceExpanded, error) {
 	clusterRGName := stringutils.LastTokenByte(a.oc.Properties.ClusterProfile.ResourceGroupID, '/')
 
-	resources, err := a.resources.ListByResourceGroup(ctx, clusterRGName, "resourceType ne 'Microsoft.Compute/snapshots'", "", nil)
-	if err != nil {
-		return nil, err
-	}
+	return a.resources.ListByResourceGroup(ctx, clusterRGName, "", "", nil)
+}
+
+func (a *azureActions) ResourcesList(ctx context.Context, resources []mgmtfeatures.GenericResourceExpanded, writer io.WriteCloser) error {
+	defer writer.Close()
 
+	clusterRGName := stringutils.LastTokenByte(a.oc.Properties.ClusterProfile.ResourceGroupID, '/')
+	writer.Write([]byte("["))
 	// +4 because we expect +2 subnet and +1 vnets and optional +1 diskEncryptionSet
-	armResources := make([]arm.Resource, 0, len(resources)+4)
-	armResources, err = a.appendAzureNetworkResources(ctx, armResources)
+	armResources := make([]arm.Resource, 0, 4)
+	armResources, err := a.appendAzureNetworkResources(ctx, armResources)
 	if err != nil {
 		a.log.Warnf("error when getting network resources: %s", err)
 	}
@@ -34,6 +40,15 @@ func (a *azureActions) ResourcesList(ctx context.Context) ([]byte, error) {
 	if err != nil {
 		a.log.Warnf("error when getting DiskEncryptionSet resources: %s", err)
 	}
+	hasWritten := false
+
+	for _, resource := range armResources {
+		if hasWritten {
+			writer.Write(commaAsByteArray)
+		}
+		a.writeObject(writer, resource)
+		hasWritten = true
+	}
 
 	for _, res := range resources {
 		apiVersion := azureclient.APIVersion(*res.Type)
@@ -44,35 +59,57 @@ func (a *azureActions) ResourcesList(ctx context.Context) ([]byte, error) {
 			a.log.Warnf("API version not found for type %q", *res.Type)
 			continue
 		}
+
+		if hasWritten {
+			writer.Write(commaAsByteArray)
+		}
+		hasWritten = true
+
 		switch *res.Type {
 		case "Microsoft.Compute/virtualMachines":
 			vm, err := a.virtualMachines.Get(ctx, clusterRGName, *res.Name, mgmtcompute.InstanceView)
 			if err != nil {
 				a.log.Warn(err) // can happen when the ARM cache is lagging
-				armResources = append(armResources, arm.Resource{
+				a.writeObject(writer, arm.Resource{
 					Resource: res,
 				})
 				continue
 			}
-			armResources = append(armResources, arm.Resource{
+			a.writeObject(writer, arm.Resource{
 				Resource: vm,
 			})
 		default:
 			gr, err := a.resources.GetByID(ctx, *res.ID, apiVersion)
 			if err != nil {
 				a.log.Warn(err) // can happen when the ARM cache is lagging
-				armResources = append(armResources, arm.Resource{
+				a.writeObject(writer, arm.Resource{
 					Resource: res,
 				})
 				continue
 			}
-			armResources = append(armResources, arm.Resource{
+			a.writeObject(writer, arm.Resource{
 				Resource: gr,
 			})
 		}
 	}
 
-	return json.Marshal(armResources)
+	_, err = writer.Write([]byte("]"))
+
+	return err
+}
+
+func (a *azureActions) writeObject(writer io.Writer, resource arm.Resource) {
+	bytes, err := resource.MarshalJSON()
+	if err != nil {
+		a.log.Warn(err) // very unlikely , only a handful of cases trigger an error
+		// here. and since we get the object from a database , it probably will never happen
+		return
+	}
+
+	_, err = writer.Write(bytes)
+	if err != nil {
+		a.log.Warn(err) // can happen if the the connection is closed for example
+	}
 }
 
 func (a *azureActions) appendAzureNetworkResources(ctx context.Context, armResources []arm.Resource) ([]arm.Resource, error) {
@@ -147,3 +184,9 @@ func (a *azureActions) appendAzureDiskEncryptionSetResources(ctx context.Context
 
 	return armResources, nil
 }
+
+func (a *azureActions) WriteToStream(ctx context.Context, writer io.WriteCloser) error {
+	resources, err := a.GroupResourceList(ctx)
+	go a.ResourcesList(ctx, resources, writer)
+	return err
+}
diff --git a/pkg/frontend/adminreplies.go b/pkg/frontend/adminreplies.go
new file mode 100644
index 000000000..375d94eb7
--- /dev/null
+++ b/pkg/frontend/adminreplies.go
@@ -0,0 +1,75 @@
+package frontend
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/sirupsen/logrus"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+)
+
+type StreamResponder interface {
+	ReplyStream(log *logrus.Entry, w http.ResponseWriter, header http.Header, reader io.Reader, err error)
+	AdminReplyStream(log *logrus.Entry, w http.ResponseWriter, header http.Header, reader io.Reader, err error)
+}
+
+type defaultResponder struct {
+}
+
+func (d defaultResponder) ReplyStream(log *logrus.Entry, w http.ResponseWriter, header http.Header, reader io.Reader, err error) {
+	for k, v := range header {
+		w.Header()[k] = v
+	}
+
+	if err != nil {
+		switch err := err.(type) {
+		case *api.CloudError:
+			log.Info(err)
+			api.WriteCloudError(w, err)
+			return
+		case statusCodeError:
+			w.WriteHeader(int(err))
+		default:
+			log.Error(err)
+			api.WriteError(w, http.StatusInternalServerError, api.CloudErrorCodeInternalServerError, "", "Internal server error.")
+			return
+		}
+	}
+
+	io.Copy(w, reader)
+	_, _ = w.Write([]byte{'\n'})
+}
+
+func (d defaultResponder) AdminReplyStream(log *logrus.Entry, w http.ResponseWriter, header http.Header, reader io.Reader, err error) {
+	if apiErr, ok := err.(kerrors.APIStatus); ok {
+		status := apiErr.Status()
+
+		var target string
+		if status.Details != nil {
+			gk := schema.GroupKind{
+				Group: status.Details.Group,
+				Kind:  status.Details.Kind,
+			}
+
+			target = fmt.Sprintf("%s/%s", gk, status.Details.Name)
+		}
+
+		err = &api.CloudError{
+			StatusCode: int(status.Code),
+			CloudErrorBody: &api.CloudErrorBody{
+				Code:    string(status.Reason),
+				Message: status.Message,
+				Target:  target,
+			},
+		}
+	}
+
+	d.ReplyStream(log, w, header, reader, err)
+}
diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go
index 7db7f5255..b60372c9a 100644
--- a/pkg/frontend/frontend.go
+++ b/pkg/frontend/frontend.go
@@ -30,6 +30,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/clusterdata"
 	"github.com/Azure/ARO-RP/pkg/util/encryption"
 	"github.com/Azure/ARO-RP/pkg/util/heartbeat"
+	utillog "github.com/Azure/ARO-RP/pkg/util/log"
 	"github.com/Azure/ARO-RP/pkg/util/recover"
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
@@ -88,6 +89,8 @@ type frontend struct {
 	systemDataMachinePoolEnricher          func(*api.ClusterManagerConfigurationDocument, *api.SystemData)
 	systemDataSyncIdentityProviderEnricher func(*api.ClusterManagerConfigurationDocument, *api.SystemData)
 	systemDataSecretEnricher               func(*api.ClusterManagerConfigurationDocument, *api.SystemData)
+
+	streamResponder StreamResponder
 }
 
 // Runnable represents a runnable object
@@ -149,6 +152,8 @@ func NewFrontend(ctx context.Context,
 		systemDataMachinePoolEnricher:          enrichMachinePoolSystemData,
 		systemDataSyncIdentityProviderEnricher: enrichSyncIdentityProviderSystemData,
 		systemDataSecretEnricher:               enrichSecretSystemData,
+
+		streamResponder: defaultResponder{},
 	}
 
 	l, err := f.env.Listen()
@@ -517,3 +522,35 @@ func reply(log *logrus.Entry, w http.ResponseWriter, header http.Header, b []byt
 		_, _ = w.Write([]byte{'\n'})
 	}
 }
+
+func frontendOperationResultLog(log *logrus.Entry, method string, err error) {
+	log = log.WithFields(logrus.Fields{
+		"LOGKIND":       "frontendqos",
+		"resultType":    utillog.SuccessResultType,
+		"operationType": method,
+	})
+
+	if err == nil {
+		log.Info("front end operation succeeded")
+		return
+	}
+
+	switch err := err.(type) {
+	case *api.CloudError:
+		log = log.WithField("resultType", utillog.UserErrorResultType)
+	case statusCodeError:
+		if int(err) < 300 && int(err) >= 200 {
+			log.Info("front end operation succeeded")
+			return
+		} else if int(err) < 500 {
+			log = log.WithField("resultType", utillog.UserErrorResultType)
+		} else {
+			log = log.WithField("resultType", utillog.ServerErrorResultType)
+		}
+	default:
+		log = log.WithField("resultType", utillog.ServerErrorResultType)
+	}
+
+	log = log.WithField("errorDetails", err.Error())
+	log.Info("front end operation failed")
+}
diff --git a/pkg/frontend/generate.go b/pkg/frontend/generate.go
index 5ece7c7bc..bd29789f3 100644
--- a/pkg/frontend/generate.go
+++ b/pkg/frontend/generate.go
@@ -4,4 +4,5 @@ package frontend
 // Licensed under the Apache License 2.0.
 
 //go:generate rm -rf ../../util/mocks/$GOPACKAGE
-//go:generate go run ../../vendor/github.com/golang/mock/mockgen -destination=../util/mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/frontend QuotaValidator
+//go:generate go run ../../vendor/github.com/golang/mock/mockgen -destination=../util/mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/frontend StreamResponder,QuotaValidator
+//go:generate go run ../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../util/mocks/$GOPACKAGE/$GOPACKAGE.go
diff --git a/pkg/frontend/openshiftcluster_delete.go b/pkg/frontend/openshiftcluster_delete.go
index 59babae0e..8534a2f2b 100644
--- a/pkg/frontend/openshiftcluster_delete.go
+++ b/pkg/frontend/openshiftcluster_delete.go
@@ -31,6 +31,7 @@ func (f *frontend) deleteOpenShiftCluster(w http.ResponseWriter, r *http.Request
 		err = statusCodeError(http.StatusAccepted)
 	}
 
+	frontendOperationResultLog(log, r.Method, err)
 	reply(log, w, header, nil, err)
 }
 
diff --git a/pkg/frontend/openshiftcluster_get.go b/pkg/frontend/openshiftcluster_get.go
index 3b0d783cc..2cf9c021e 100644
--- a/pkg/frontend/openshiftcluster_get.go
+++ b/pkg/frontend/openshiftcluster_get.go
@@ -24,6 +24,7 @@ func (f *frontend) getOpenShiftCluster(w http.ResponseWriter, r *http.Request) {
 
 	b, err := f._getOpenShiftCluster(ctx, log, r, f.apis[vars["api-version"]].OpenShiftClusterConverter)
 
+	frontendOperationResultLog(log, r.Method, err)
 	reply(log, w, nil, b, err)
 }
 
diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go
index 054118660..f44baab32 100644
--- a/pkg/frontend/openshiftcluster_putorpatch.go
+++ b/pkg/frontend/openshiftcluster_putorpatch.go
@@ -44,6 +44,7 @@ func (f *frontend) putOrPatchOpenShiftCluster(w http.ResponseWriter, r *http.Req
 		return err
 	})
 
+	frontendOperationResultLog(log, r.Method, err)
 	reply(log, w, header, b, err)
 }
 
diff --git a/pkg/util/mocks/adminactions/adminactions.go b/pkg/util/mocks/adminactions/adminactions.go
index d90f6302d..42ad1dc9c 100644
--- a/pkg/util/mocks/adminactions/adminactions.go
+++ b/pkg/util/mocks/adminactions/adminactions.go
@@ -6,10 +6,12 @@ package mock_adminactions
 
 import (
 	context "context"
+	io "io"
 	http "net/http"
 	reflect "reflect"
 
 	compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
+	features "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
 	gomock "github.com/golang/mock/gomock"
 	logrus "github.com/sirupsen/logrus"
 	unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -234,6 +236,21 @@ func (mr *MockAzureActionsMockRecorder) AppLensListDetectors(arg0 interface{}) *
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppLensListDetectors", reflect.TypeOf((*MockAzureActions)(nil).AppLensListDetectors), arg0)
 }
 
+// GroupResourceList mocks base method.
+func (m *MockAzureActions) GroupResourceList(arg0 context.Context) ([]features.GenericResourceExpanded, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GroupResourceList", arg0)
+	ret0, _ := ret[0].([]features.GenericResourceExpanded)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GroupResourceList indicates an expected call of GroupResourceList.
+func (mr *MockAzureActionsMockRecorder) GroupResourceList(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GroupResourceList", reflect.TypeOf((*MockAzureActions)(nil).GroupResourceList), arg0)
+}
+
 // NICReconcileFailedState mocks base method.
 func (m *MockAzureActions) NICReconcileFailedState(arg0 context.Context, arg1 string) error {
 	m.ctrl.T.Helper()
@@ -249,18 +266,17 @@ func (mr *MockAzureActionsMockRecorder) NICReconcileFailedState(arg0, arg1 inter
 }
 
 // ResourcesList mocks base method.
-func (m *MockAzureActions) ResourcesList(arg0 context.Context) ([]byte, error) {
+func (m *MockAzureActions) ResourcesList(arg0 context.Context, arg1 []features.GenericResourceExpanded, arg2 io.WriteCloser) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "ResourcesList", arg0)
-	ret0, _ := ret[0].([]byte)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
+	ret := m.ctrl.Call(m, "ResourcesList", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
 }
 
 // ResourcesList indicates an expected call of ResourcesList.
-func (mr *MockAzureActionsMockRecorder) ResourcesList(arg0 interface{}) *gomock.Call {
+func (mr *MockAzureActionsMockRecorder) ResourcesList(arg0, arg1, arg2 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourcesList", reflect.TypeOf((*MockAzureActions)(nil).ResourcesList), arg0)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourcesList", reflect.TypeOf((*MockAzureActions)(nil).ResourcesList), arg0, arg1, arg2)
 }
 
 // VMRedeployAndWait mocks base method.
@@ -347,3 +363,17 @@ func (mr *MockAzureActionsMockRecorder) VMStopAndWait(arg0, arg1 interface{}) *g
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VMStopAndWait", reflect.TypeOf((*MockAzureActions)(nil).VMStopAndWait), arg0, arg1)
 }
+
+// WriteToStream mocks base method.
+func (m *MockAzureActions) WriteToStream(arg0 context.Context, arg1 io.WriteCloser) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "WriteToStream", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// WriteToStream indicates an expected call of WriteToStream.
+func (mr *MockAzureActionsMockRecorder) WriteToStream(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteToStream", reflect.TypeOf((*MockAzureActions)(nil).WriteToStream), arg0, arg1)
+}
diff --git a/pkg/util/mocks/frontend/frontend.go b/pkg/util/mocks/frontend/frontend.go
index 62474ce2b..2cca357c8 100644
--- a/pkg/util/mocks/frontend/frontend.go
+++ b/pkg/util/mocks/frontend/frontend.go
@@ -1,19 +1,70 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/frontend (interfaces: QuotaValidator)
+// Source: github.com/Azure/ARO-RP/pkg/frontend (interfaces: StreamResponder,QuotaValidator)
 
 // Package mock_frontend is a generated GoMock package.
 package mock_frontend
 
 import (
 	context "context"
+	io "io"
+	http "net/http"
 	reflect "reflect"
 
+	gomock "github.com/golang/mock/gomock"
+	logrus "github.com/sirupsen/logrus"
+
 	api "github.com/Azure/ARO-RP/pkg/api"
 	env "github.com/Azure/ARO-RP/pkg/env"
 	azureclient "github.com/Azure/ARO-RP/pkg/util/azureclient"
-	gomock "github.com/golang/mock/gomock"
 )
 
+// MockStreamResponder is a mock of StreamResponder interface.
+type MockStreamResponder struct {
+	ctrl     *gomock.Controller
+	recorder *MockStreamResponderMockRecorder
+}
+
+// MockStreamResponderMockRecorder is the mock recorder for MockStreamResponder.
+type MockStreamResponderMockRecorder struct {
+	mock *MockStreamResponder
+}
+
+// NewMockStreamResponder creates a new mock instance.
+func NewMockStreamResponder(ctrl *gomock.Controller) *MockStreamResponder {
+	mock := &MockStreamResponder{ctrl: ctrl}
+	mock.recorder = &MockStreamResponderMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockStreamResponder) EXPECT() *MockStreamResponderMockRecorder {
+	return m.recorder
+}
+
+// AdminReplyStream mocks base method.
+func (m *MockStreamResponder) AdminReplyStream(arg0 *logrus.Entry, arg1 http.ResponseWriter, arg2 http.Header, arg3 io.Reader, arg4 error) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "AdminReplyStream", arg0, arg1, arg2, arg3, arg4)
+}
+
+// AdminReplyStream indicates an expected call of AdminReplyStream.
+func (mr *MockStreamResponderMockRecorder) AdminReplyStream(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdminReplyStream", reflect.TypeOf((*MockStreamResponder)(nil).AdminReplyStream), arg0, arg1, arg2, arg3, arg4)
+}
+
+// ReplyStream mocks base method.
+func (m *MockStreamResponder) ReplyStream(arg0 *logrus.Entry, arg1 http.ResponseWriter, arg2 http.Header, arg3 io.Reader, arg4 error) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "ReplyStream", arg0, arg1, arg2, arg3, arg4)
+}
+
+// ReplyStream indicates an expected call of ReplyStream.
+func (mr *MockStreamResponderMockRecorder) ReplyStream(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReplyStream", reflect.TypeOf((*MockStreamResponder)(nil).ReplyStream), arg0, arg1, arg2, arg3, arg4)
+}
+
 // MockQuotaValidator is a mock of QuotaValidator interface.
 type MockQuotaValidator struct {
 	ctrl     *gomock.Controller


Test patch:
diff --git a/pkg/frontend/admin_openshiftcluster_resources_list_test.go b/pkg/frontend/admin_openshiftcluster_resources_list_test.go
index af4ba9f70..de920e70e 100644
--- a/pkg/frontend/admin_openshiftcluster_resources_list_test.go
+++ b/pkg/frontend/admin_openshiftcluster_resources_list_test.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
 	"github.com/golang/mock/gomock"
 	"github.com/sirupsen/logrus"
 
@@ -18,6 +19,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/frontend/adminactions"
 	"github.com/Azure/ARO-RP/pkg/metrics/noop"
 	mock_adminactions "github.com/Azure/ARO-RP/pkg/util/mocks/adminactions"
+	mock_frontend "github.com/Azure/ARO-RP/pkg/util/mocks/frontend"
 	testdatabase "github.com/Azure/ARO-RP/test/database"
 )
 
@@ -32,8 +34,6 @@ func TestAdminListResourcesList(t *testing.T) {
 		fixture        func(f *testdatabase.Fixture)
 		mocks          func(*test, *mock_adminactions.MockAzureActions)
 		wantStatusCode int
-		wantResponse   []byte
-		wantError      string
 	}
 
 	for _, tt := range []*test{
@@ -67,11 +67,18 @@ func TestAdminListResourcesList(t *testing.T) {
 			},
 			mocks: func(tt *test, a *mock_adminactions.MockAzureActions) {
 				a.EXPECT().
-					ResourcesList(gomock.Any()).
-					Return([]byte(`[{"properties":{"dhcpOptions":{"dnsServers":[]}},"id":"/subscriptions/id","type":"Microsoft.Network/virtualNetworks"},{"properties":{"provisioningState":"Succeeded"},"id":"/subscriptions/id","type":"Microsoft.Compute/virtualMachines"},{"id":"/subscriptions/id","name":"storage","type":"Microsoft.Storage/storageAccounts","location":"eastus"}]`), nil)
+					ResourcesList(gomock.Any(), gomock.Any(), gomock.Any()).
+					Return(nil).
+					AnyTimes()
+				a.EXPECT().
+					GroupResourceList(gomock.Any()).
+					Return([]mgmtfeatures.GenericResourceExpanded{}, nil).
+					AnyTimes()
+				a.EXPECT().
+					WriteToStream(gomock.Any(), gomock.Any()).
+					Return(nil)
 			},
 			wantStatusCode: http.StatusOK,
-			wantResponse:   []byte(`[{"properties":{"dhcpOptions":{"dnsServers":[]}},"id":"/subscriptions/id","type":"Microsoft.Network/virtualNetworks"},{"properties":{"provisioningState":"Succeeded"},"id":"/subscriptions/id","type":"Microsoft.Compute/virtualMachines"},{"id":"/subscriptions/id","name":"storage","type":"Microsoft.Storage/storageAccounts","location":"eastus"}]` + "\n"),
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
@@ -89,6 +96,9 @@ func TestAdminListResourcesList(t *testing.T) {
 			f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, nil, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster, *api.SubscriptionDocument) (adminactions.AzureActions, error) {
 				return a, nil
 			}, nil)
+			mockResponder := mock_frontend.NewMockStreamResponder(ti.controller)
+			mockResponder.EXPECT().AdminReplyStream(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any())
+			f.streamResponder = mockResponder
 
 			if err != nil {
 				t.Fatal(err)
@@ -96,17 +106,12 @@ func TestAdminListResourcesList(t *testing.T) {
 
 			go f.Run(ctx, nil, nil)
 
-			resp, b, err := ti.request(http.MethodGet,
+			_, _, err = ti.request(http.MethodGet,
 				fmt.Sprintf("https://server/admin/%s/resources", tt.resourceID),
 				nil, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
-
-			err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse)
-			if err != nil {
-				t.Error(err)
-			}
 		})
 	}
 }
diff --git a/pkg/frontend/adminactions/resources_list_test.go b/pkg/frontend/adminactions/resources_list_test.go
index 521cd14d3..9c813a50e 100644
--- a/pkg/frontend/adminactions/resources_list_test.go
+++ b/pkg/frontend/adminactions/resources_list_test.go
@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"testing"
 
 	mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
@@ -25,7 +26,7 @@ import (
 )
 
 func validListByResourceGroupMock(resources *mock_features.MockResourcesClient) {
-	resources.EXPECT().ListByResourceGroup(gomock.Any(), "test-cluster", "resourceType ne 'Microsoft.Compute/snapshots'", "", nil).Return([]mgmtfeatures.GenericResourceExpanded{
+	resources.EXPECT().ListByResourceGroup(gomock.Any(), "test-cluster", "", "", nil).Return([]mgmtfeatures.GenericResourceExpanded{
 		{
 			Name: to.StringPtr("vm-1"),
 			ID:   to.StringPtr("/subscriptions/id"),
@@ -53,7 +54,7 @@ func validVirtualMachinesMock(virtualMachines *mock_compute.MockVirtualMachinesC
 		VirtualMachineProperties: &mgmtcompute.VirtualMachineProperties{
 			ProvisioningState: to.StringPtr("Succeeded"),
 		},
-	}, nil)
+	}, nil).AnyTimes()
 }
 
 func validVirtualNetworksMock(virtualNetworks *mock_network.MockVirtualNetworksClient, routeTables *mock_network.MockRouteTablesClient, mockSubID string) {
@@ -179,12 +180,14 @@ func TestResourcesList(t *testing.T) {
 				routeTables:        routeTables,
 			}
 
-			b, err := a.ResourcesList(ctx)
+			reader, writer := io.Pipe()
+			err := a.WriteToStream(ctx, writer)
 
+			b, _ := io.ReadAll(reader)
 			if tt.wantError == "" {
 				if tt.wantResponse != nil {
 					if !bytes.Equal(b, tt.wantResponse) {
-						t.Error(string(b))
+						t.Errorf("wanted %s got %s", string(tt.wantResponse), string(b))
 					}
 				}
 			} else {