Repository: Azure/ARO-RP
Pull request #2109 (2022-05-03 23:00:44)

Add Approve CSR Admin Endpoint for New Geneva Action
This PR creates an admin endpoint that can be used to approve CSR's in an ARO cluster. New endpoint will be targeted by a new Geneva Action (Phase 3)

### Which issue this PR addresses:

[USER STORY 13911981 - Create Cordon or Uncordon Node Geneva Action](https://dev.azure.com/msazure/AzureRedHatOpenShift/_workitems/edit/13911981)

### What this PR does / why we need it:

This PR add an admin endpoint that can be used to approve CSR's in an ARO cluster. CSR's can be approved individually by name, or all CSR's can be approved at once. This endpoint will eventually be targeted by a new Geneva Action that has been requested by ARO support (Phase 3 Geneva Actions).

### Test plan for issue:

Unit tests have been added for the new endpoint. I also ran the RP locally, created a cluster, and manually curled the new "approveCsr" endpoint to confirm...
- the approval of a single CSR in the cluster
- mass approval of CSR's in the cluster
- invalid parameters return expected errors

### Is there any documentation that needs to be updated for this PR?

No - once the Geneva Action frontends have been created, that documentation will be updated to include the functionality of these new endpoints.

github-actions[bot]: Please rebase pull request.
petrkotas: /azp run ci
azure-pipelines[bot]: <samp>
Pull request contains merge conflicts.<br>

</samp>
zacharyljones: I also changed...

`approveAll := r.URL.Query().Get("approveAll") == "true"`

to

`approveAll := strings.EqualFold(r.URL.Query().Get("approveAll"), "true")`

since I have received similar comments on other PRs.
zacharyljones: /azp run ci
azure-pipelines[bot]: <samp>
Commenter does not have sufficient privileges for PR 2109 in repo Azure/ARO-RP<br>

</samp>
zacharyljones: > /azp run ci

This was a test
facchettos: /azp run ci e2e
azure-pipelines[bot]: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
hawkowl: @zacharyljones I think merging your other PR causes conflicts in this one. Could you rebase?
github-actions[bot]: Please rebase pull request.
zacharyljones: Hold - waiting for addition of e2e tests from Damian
dlmiller322: @bennerv Updated the approvecsr admin endpoint to reflect your requested changes from the meeting earlier. Lemme know what you think.
![image](https://user-images.githubusercontent.com/83246827/185983574-5c9751e2-b0f6-4a99-8da0-e9cd93d5d73d.png)


Patch:
diff --git a/pkg/frontend/admin_openshiftcluster_approvecsr.go b/pkg/frontend/admin_openshiftcluster_approvecsr.go
new file mode 100644
index 000000000..7aef1c81e
--- /dev/null
+++ b/pkg/frontend/admin_openshiftcluster_approvecsr.go
@@ -0,0 +1,61 @@
+package frontend
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/database/cosmosdb"
+	"github.com/Azure/ARO-RP/pkg/frontend/middleware"
+)
+
+func (f *frontend) postAdminOpenShiftClusterApproveCSR(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry)
+	r.URL.Path = filepath.Dir(r.URL.Path)
+
+	err := f._postAdminOpenShiftClusterApproveCSR(ctx, r, log)
+
+	adminReply(log, w, nil, nil, err)
+}
+
+func (f *frontend) _postAdminOpenShiftClusterApproveCSR(ctx context.Context, r *http.Request, log *logrus.Entry) error {
+	vars := mux.Vars(r)
+
+	csrName := r.URL.Query().Get("csrName")
+	if csrName != "" {
+		err := validateAdminKubernetesObjects(r.Method, "CertificateSigningRequest", "", csrName)
+		if err != nil {
+			return err
+		}
+	}
+
+	resourceID := strings.TrimPrefix(r.URL.Path, "/admin")
+
+	doc, err := f.dbOpenShiftClusters.Get(ctx, resourceID)
+	switch {
+	case cosmosdb.IsErrorStatusCode(err, http.StatusNotFound):
+		return api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeResourceNotFound, "", "The Resource '%s/%s' under resource group '%s' was not found.", vars["resourceType"], vars["resourceName"], vars["resourceGroupName"])
+	case err != nil:
+		return err
+	}
+
+	k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster)
+	if err != nil {
+		return err
+	}
+
+	if csrName != "" {
+		return k.ApproveCsr(ctx, csrName)
+	}
+
+	return k.ApproveAllCsrs(ctx)
+}
diff --git a/pkg/frontend/adminactions/certificatesigningrequests.go b/pkg/frontend/adminactions/certificatesigningrequests.go
new file mode 100644
index 000000000..aec407708
--- /dev/null
+++ b/pkg/frontend/adminactions/certificatesigningrequests.go
@@ -0,0 +1,77 @@
+package adminactions
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"net/http"
+
+	certificatesv1 "k8s.io/api/certificates/v1"
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+)
+
+func (k *kubeActions) ApproveCsr(ctx context.Context, csrName string) error {
+	csr, err := k.kubecli.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{})
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeResourceNotFound, "", "certificate signing request '%s' was not found.", csrName)
+		}
+		return err
+	}
+
+	return k.updateCsr(ctx, csr)
+}
+
+func (k *kubeActions) ApproveAllCsrs(ctx context.Context) error {
+	csrs, err := k.kubecli.CertificatesV1().CertificateSigningRequests().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, csr := range csrs.Items {
+		err = k.updateCsr(ctx, &csr)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (k *kubeActions) updateCsr(ctx context.Context, csr *certificatesv1.CertificateSigningRequest) error {
+	modifiedCSR, hasCondition, err := addConditionIfNeeded(csr, string(certificatesv1.CertificateDenied), string(certificatesv1.CertificateApproved), "AROSupportApprove", "This CSR was approved by ARO support personnel.")
+
+	if err != nil {
+		return err
+	}
+	if !hasCondition {
+		_, err = k.kubecli.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, modifiedCSR.Name, modifiedCSR, metav1.UpdateOptions{})
+	}
+	return err
+}
+
+func addConditionIfNeeded(csr *certificatesv1.CertificateSigningRequest, mustNotHaveConditionType, conditionType, reason, message string) (*certificatesv1.CertificateSigningRequest, bool, error) {
+	var alreadyHasCondition bool
+	for _, c := range csr.Status.Conditions {
+		if string(c.Type) == mustNotHaveConditionType {
+			return nil, false, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodePropertyChangeNotAllowed, "", "certificate signing request %q is already %s", csr.Name, c.Type)
+		}
+		if string(c.Type) == conditionType {
+			alreadyHasCondition = true
+		}
+	}
+	if alreadyHasCondition {
+		return csr, true, nil
+	}
+	csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{
+		Type:           certificatesv1.RequestConditionType(conditionType),
+		Status:         corev1.ConditionTrue,
+		Reason:         reason,
+		Message:        message,
+		LastUpdateTime: metav1.Now(),
+	})
+	return csr, false, nil
+}
diff --git a/pkg/frontend/adminactions/kubeactions.go b/pkg/frontend/adminactions/kubeactions.go
index 0180121b9..a63bb63a1 100644
--- a/pkg/frontend/adminactions/kubeactions.go
+++ b/pkg/frontend/adminactions/kubeactions.go
@@ -31,6 +31,8 @@ type KubeActions interface {
 	KubeDelete(ctx context.Context, groupKind, namespace, name string, force bool) error
 	CordonNode(ctx context.Context, nodeName string, unschedulable bool) error
 	DrainNode(ctx context.Context, nodeName string) error
+	ApproveCsr(ctx context.Context, csrName string) error
+	ApproveAllCsrs(ctx context.Context) error
 	Upgrade(ctx context.Context, upgradeY bool) error
 	KubeGetPodLogs(ctx context.Context, namespace, name, containerName string) ([]byte, error)
 }
diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go
index 568c089e5..4687c4cb3 100644
--- a/pkg/frontend/frontend.go
+++ b/pkg/frontend/frontend.go
@@ -256,6 +256,12 @@ func (f *frontend) authenticatedRoutes(r *mux.Router) {
 	s.Methods(http.MethodPost).HandlerFunc(f.postAdminKubernetesObjects).Name("postAdminKubernetesObjects")
 	s.Methods(http.MethodDelete).HandlerFunc(f.deleteAdminKubernetesObjects).Name("deleteAdminKubernetesObjects")
 
+	s = r.
+		Path("/admin/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/approvecsr").
+		Subrouter()
+
+	s.Methods(http.MethodPost).HandlerFunc(f.postAdminOpenShiftClusterApproveCSR).Name("postAdminOpenShiftClusterApproveCSR")
+
 	// Pod logs
 	s = r.
 		Path("/admin/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/kubernetespodlogs").
diff --git a/pkg/util/mocks/adminactions/adminactions.go b/pkg/util/mocks/adminactions/adminactions.go
index 75e0d6514..8e7b06491 100644
--- a/pkg/util/mocks/adminactions/adminactions.go
+++ b/pkg/util/mocks/adminactions/adminactions.go
@@ -38,6 +38,34 @@ func (m *MockKubeActions) EXPECT() *MockKubeActionsMockRecorder {
 	return m.recorder
 }
 
+// ApproveAllCsrs mocks base method.
+func (m *MockKubeActions) ApproveAllCsrs(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ApproveAllCsrs", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ApproveAllCsrs indicates an expected call of ApproveAllCsrs.
+func (mr *MockKubeActionsMockRecorder) ApproveAllCsrs(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApproveAllCsrs", reflect.TypeOf((*MockKubeActions)(nil).ApproveAllCsrs), arg0)
+}
+
+// ApproveCsr mocks base method.
+func (m *MockKubeActions) ApproveCsr(arg0 context.Context, arg1 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ApproveCsr", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// ApproveCsr indicates an expected call of ApproveCsr.
+func (mr *MockKubeActionsMockRecorder) ApproveCsr(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApproveCsr", reflect.TypeOf((*MockKubeActions)(nil).ApproveCsr), arg0, arg1)
+}
+
 // CordonNode mocks base method.
 func (m *MockKubeActions) CordonNode(arg0 context.Context, arg1 string, arg2 bool) error {
 	m.ctrl.T.Helper()


Test patch:
diff --git a/pkg/frontend/admin_openshiftcluster_approvecsr_test.go b/pkg/frontend/admin_openshiftcluster_approvecsr_test.go
new file mode 100644
index 000000000..5de793122
--- /dev/null
+++ b/pkg/frontend/admin_openshiftcluster_approvecsr_test.go
@@ -0,0 +1,117 @@
+package frontend
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/sirupsen/logrus"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/frontend/adminactions"
+	"github.com/Azure/ARO-RP/pkg/metrics/noop"
+	mock_adminactions "github.com/Azure/ARO-RP/pkg/util/mocks/adminactions"
+)
+
+func TestAdminApproveCSR(t *testing.T) {
+	mockSubID := "00000000-0000-0000-0000-000000000000"
+	mockTenantID := "00000000-0000-0000-0000-000000000000"
+	ctx := context.Background()
+
+	type test struct {
+		name           string
+		resourceID     string
+		csrName        string
+		mocks          func(*test, *mock_adminactions.MockKubeActions)
+		method         string
+		wantStatusCode int
+		wantResponse   []byte
+		wantError      string
+	}
+
+	for _, tt := range []*test{
+		{
+			method:     http.MethodPost,
+			name:       "single csr",
+			resourceID: fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID),
+			csrName:    "aro-csr",
+			mocks: func(tt *test, k *mock_adminactions.MockKubeActions) {
+				k.EXPECT().
+					ApproveCsr(gomock.Any(), tt.csrName).
+					Return(nil)
+			},
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			method:     http.MethodPost,
+			name:       "all csrs",
+			resourceID: fmt.Sprintf("/subscriptions/%s/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName", mockSubID),
+			mocks: func(tt *test, k *mock_adminactions.MockKubeActions) {
+				k.EXPECT().
+					ApproveAllCsrs(gomock.Any()).
+					Return(nil)
+			},
+			wantStatusCode: http.StatusOK,
+		},
+	} {
+		t.Run(fmt.Sprintf("%s: %s", tt.method, tt.name), func(t *testing.T) {
+			ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions()
+			defer ti.done()
+
+			k := mock_adminactions.NewMockKubeActions(ti.controller)
+			tt.mocks(tt, k)
+
+			ti.fixture.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
+				Key: strings.ToLower(tt.resourceID),
+				OpenShiftCluster: &api.OpenShiftCluster{
+					ID:   tt.resourceID,
+					Name: "resourceName",
+					Type: "Microsoft.RedHatOpenShift/openshiftClusters",
+				},
+			})
+			ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{
+				ID: mockSubID,
+				Subscription: &api.Subscription{
+					State: api.SubscriptionStateRegistered,
+					Properties: &api.SubscriptionProperties{
+						TenantID: mockTenantID,
+					},
+				},
+			})
+
+			err := ti.buildFixtures(nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.clusterManagerDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, nil, api.APIs, &noop.Noop{}, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) {
+				return k, nil
+			}, nil, nil)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			go f.Run(ctx, nil, nil)
+
+			resp, b, err := ti.request(tt.method,
+				fmt.Sprintf("https://server/admin%s/approvecsr?csrName=%s", tt.resourceID, tt.csrName),
+				nil, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse)
+			if err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
diff --git a/test/e2e/adminapi_csr.go b/test/e2e/adminapi_csr.go
new file mode 100644
index 000000000..214fde555
--- /dev/null
+++ b/test/e2e/adminapi_csr.go
@@ -0,0 +1,123 @@
+package e2e
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"encoding/base64"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	certificatesv1 "k8s.io/api/certificates/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("[Admin API] CertificateSigningRequest action", func() {
+	BeforeEach(skipIfNotInDevelopmentEnv)
+
+	const prefix = "e2e-test-csr"
+	const namespace = "openshift"
+	const csrCount = 4
+	const csrdataStr = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0KTUlJQ3BEQ0NBWXdDQVFBd1h6RUxNQWtHQTFVRUJoTUNWVk14Q3pBSkJnTlZCQWdNQWtOUE1ROHdEUVlEVlFRSApEQVpFWlc1MlpYSXhFakFRQmdOVkJBb01DVTFwWTNKdmMyOW1kREVNTUFvR0ExVUVDd3dEUVZKUE1SQXdEZ1lEClZRUUREQWRsTW1VdVlYSnZNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQWxqWUUKcnFkU0hvV1p2MVdHSnN1RFZsaGExVU1BVnJiUk0xWjJHaWZJMzNETlBUWGFmQnN1QVI2ZGVCQVgyWmpybUozNQpuekNBZ0k5d1ltdlYwN3JtTEFYQlloRnJiTWtNN1pSU1ZFT01hL2ZXdlN5ZjJVQWxSdm5Jd0JmRkgwS1pRSGg5Cm5aV3RIZHQxSzRuZ3ZnM1NuQ3JEU0NBRUhsS2hoN3Jua1pyRkdrMldabFFoVklWUXFReFFzdmx3VStvWlhnNjQKdmpleDRuc3BZaXFXMERzakl6RzFsSEszWHczN3RGeWhNNzJ4SjByblBYVTRGWkJsWXUzWkVqOFVhSFBoTlcrdgpqZmg2c0hCbWFkcHpEMWRuNDJ4eXgrUGhOaCtKWTVVT3ZWWnR2MWx5UU44eEswL0VjK0Mvcm1mOWZPYmdFSkNVCm00Z3pFSXhhVGhCVURsN1JHd0lEQVFBQm9BQXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBQnYvVHdUR0JvL20KcVJVK0djZ3Bsa3I1aDlKQVdSZjNNazV3Z1o0ZmlSZm85UEVaYUxJWkZYQ0V0elNHV3JZenFjbFpZQ3JuRmUySQpzdHdNUU8yb1pQUzNvcUVIcWs5Uk0rbzRUVmtkSldjY3hKV3RMY3JoTWRwVjVMc3VMam1qRS9jeDcrbEtUZkh1Cno0eDllYzJTajhnZmV3SFowZTkzZjFTT3ZhVGFMaTQrT3JkM3FTT0NyNE5ZSGhvVDJiM0pBUFpMSmkvVEFpb1gKOUxJNFJpVXNSSWlMUm45VDZidzczM0FLMkpNMXREWU9Tc0hXdmJrZ3FDOFlHMmpYUW9LNUpZOWdTN0V5TkF6NwpjT1plbkkwK2dVeE1leUlNN2I0S05YWFQ3NmxVdHZ5M2N3LzhwVmxQU01pTDFVZ2RpMXFZMDl0MW9FMmU4YnljCm5GdWhZOW5ERU53PQotLS0tLUVORCBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0K"
+
+	It("should be able to approve one or multiple CSRs", func() {
+
+		csrDataEncoded := []byte(csrdataStr)
+		csrDataDecoded := make([]byte, base64.StdEncoding.DecodedLen(len(csrDataEncoded)))
+		csrDataLength, err := base64.StdEncoding.Decode(csrDataDecoded, csrDataEncoded)
+		Expect(err).NotTo(HaveOccurred())
+		csrData := csrDataDecoded[:csrDataLength]
+
+		By("creating mock CSRs via Kubernetes API")
+		for i := 0; i < csrCount; i++ {
+			csr := mockCSR(prefix+strconv.Itoa(i), namespace, csrData)
+			_, err := clients.Kubernetes.CertificatesV1().CertificateSigningRequests().Create(context.Background(), csr, metav1.CreateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+		}
+
+		defer func() {
+			By("deleting the mock CSRs via Kubernetes API")
+			for i := 0; i < csrCount; i++ {
+				err := clients.Kubernetes.CertificatesV1().CertificateSigningRequests().Delete(context.Background(), prefix+strconv.Itoa(i), metav1.DeleteOptions{})
+				Expect(err).NotTo(HaveOccurred())
+			}
+		}()
+
+		testCSRApproveOK(prefix+"0", namespace)
+		testCSRMassApproveOK(prefix, namespace, csrCount)
+	})
+})
+
+func testCSRApproveOK(objName, namespace string) {
+	By("approving the CSR via RP admin API")
+	params := url.Values{
+		"csrName": []string{objName},
+	}
+	resp, err := adminRequest(context.Background(), http.MethodPost, "/admin"+resourceIDFromEnv()+"/approvecsr", params, nil, nil)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(resp.StatusCode).To(Equal(http.StatusOK))
+
+	By("checking that the CSR was approved via Kubernetes API")
+	testcsr, err := clients.Kubernetes.CertificatesV1().CertificateSigningRequests().Get(context.Background(), objName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	approved := false
+	for _, condition := range testcsr.Status.Conditions {
+		if condition.Type == certificatesv1.CertificateApproved {
+			Expect(condition.Status).To(Equal(corev1.ConditionTrue))
+			Expect(condition.Reason).To(Equal("AROSupportApprove"))
+			Expect(condition.Message).To(Equal("This CSR was approved by ARO support personnel."))
+			approved = true
+		}
+	}
+	Expect(approved).Should(BeTrue())
+}
+
+func testCSRMassApproveOK(namePrefix, namespace string, csrCount int) {
+	By("approving all CSRs via RP admin API")
+	resp, err := adminRequest(context.Background(), http.MethodPost, "/admin"+resourceIDFromEnv()+"/approvecsr", nil, nil, nil)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(resp.StatusCode).To(Equal(http.StatusOK))
+
+	By("checking that all CSRs were approved via Kubernetes API")
+	for i := 1; i < csrCount; i++ {
+		testcsr, err := clients.Kubernetes.CertificatesV1().CertificateSigningRequests().Get(context.Background(), namePrefix+strconv.Itoa(i), metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		approved := false
+		for _, condition := range testcsr.Status.Conditions {
+			if condition.Type == certificatesv1.CertificateApproved {
+				Expect(condition.Status).To(Equal(corev1.ConditionTrue))
+				Expect(condition.Reason).To(Equal("AROSupportApprove"))
+				Expect(condition.Message).To(Equal("This CSR was approved by ARO support personnel."))
+				approved = true
+			}
+		}
+
+		Expect(approved).Should(BeTrue())
+	}
+}
+
+func mockCSR(objName, namespace string, csrData []byte) *certificatesv1.CertificateSigningRequest {
+	csr := &certificatesv1.CertificateSigningRequest{
+		// Username, UID, Groups will be injected by API server.
+		TypeMeta: metav1.TypeMeta{Kind: "CertificateSigningRequest"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      objName,
+			Namespace: namespace,
+		},
+		Spec: certificatesv1.CertificateSigningRequestSpec{
+			Request:    csrData,
+			Usages:     []certificatesv1.KeyUsage{certificatesv1.UsageClientAuth},
+			SignerName: "kubernetes.io/kube-apiserver-client",
+		},
+	}
+
+	return csr
+}