Repository: Azure/ARO-RP
Pull request #2103 (2022-05-02 16:07:51)

Add Cordon and Drain Related Admin Endpoints for New Geneva Actions
This PR creates admin endpoints that can be used to cordon, uncordon, and drain ARO cluster nodes. New endpoints will be targeted by new Geneva Actions (Phase 3)

### Which issue this PR addresses:

[USER STORY 13912018 - Create Cordon or Uncordon Node Geneva Action](https://dev.azure.com/msazure/AzureRedHatOpenShift/_workitems/edit/13912018)
[USER STORY 13912064 - Create Drain Node Geneva Action](https://dev.azure.com/msazure/AzureRedHatOpenShift/_workitems/edit/13912064/)

### What this PR does / why we need it:

This PR adds admin endpoints that can be used to cordon, uncordon, and drain ARO cluster nodes. These endpoints will eventually be targeted by new Geneva Actions that have been requested by ARO support (Phase 3 Geneva Actions).

### Test plan for issue:

Unit tests have been added for each new endpoint. I also ran the RP locally, created a cluster, and manually... 
- curled the new "cordonnode" endpoint to confirm the schedulability of cluster nodes updated as expected
- curled the new "drainnode" endpoint to confirm cluster nodes drained appropriately 

Note that E2E tests should be added for each endpoint as these are potentially destructive actions, however I personally have no way of testing E2E locally at the moment. CloudFit/MS/Red Hat are still working through simply secure issues affecting the ability to run E2E tests locally. No ETA on a resolution, but didn't want that to prevent getting more eyes on this PR.

Once E2E tests can be developed and tested locally in simply secure environments,  either this PR will be updated or a separate PR will be created specifically for those tests - whichever Red Hat prefers. 

### Is there any documentation that needs to be updated for this PR?

No - once the Geneva Action frontends have been created, that documentation will be updated to include the functionality of these new endpoints.


zacharyljones: Happier with this version - ready for review
zacharyljones: want to point out that I updated the timeout for the drainer for the drain node endpoint - during this latest round of testing I would hit the 60 second timeout consistently with an "empty" cluster
hawkowl: /azp run ci
azure-pipelines[bot]: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>
hawkowl: /azp run e2e
azure-pipelines[bot]: <samp>
Azure Pipelines successfully started running 1 pipeline(s).<br>

</samp>

Patch:
diff --git a/pkg/cluster/delete.go b/pkg/cluster/delete.go
index e5599c7e4..b84be6073 100644
--- a/pkg/cluster/delete.go
+++ b/pkg/cluster/delete.go
@@ -16,6 +16,7 @@ import (
 	mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/davecgh/go-spew/spew"
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/Azure/ARO-RP/pkg/api"
@@ -370,11 +371,19 @@ func (m *manager) deleteResourcesAndResourceGroup(ctx context.Context) error {
 
 	m.log.Printf("deleting resource group %s", resourceGroup)
 	err = m.resourceGroups.DeleteAndWait(ctx, resourceGroup)
-	detailedErr, ok := err.(autorest.DetailedError)
-	if ok && (detailedErr.StatusCode == http.StatusForbidden || detailedErr.StatusCode == http.StatusNotFound) {
+	// TODO(mjudeikis): Remove this once we know all the error flavors this is
+	// returning so we can unify checks bellow
+	if err != nil {
+		m.log.Printf("delete resource group failed: %s", spew.Sdump(err))
+	}
+	if detailedErr, ok := err.(autorest.DetailedError); ok &&
+		(detailedErr.StatusCode == http.StatusForbidden || detailedErr.StatusCode == http.StatusNotFound) {
+		err = nil
+	}
+	if azureerrors.HasAuthorizationFailedError(err) {
 		err = nil
 	}
-	if azureerrors.HasAuthorizationFailedError(err) || azureerrors.ResourceGroupNotFound(err) {
+	if azureerrors.ResourceGroupNotFound(err) {
 		err = nil
 	}
 	return err
diff --git a/pkg/deploy/config.go b/pkg/deploy/config.go
index 55d10a802..3d745110f 100644
--- a/pkg/deploy/config.go
+++ b/pkg/deploy/config.go
@@ -4,9 +4,6 @@ package deploy
 // Licensed under the Apache License 2.0.
 
 import (
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
 	"fmt"
 	"io/ioutil"
 	"reflect"
@@ -93,7 +90,7 @@ type Configuration struct {
 	RPNSGSourceAddressPrefixes         []string      `json:"rpNsgSourceAddressPrefixes,omitempty" value:"required"`
 	RPParentDomainName                 *string       `json:"rpParentDomainName,omitempty" value:"required"`
 	RPVMSSCapacity                     *int          `json:"rpVmssCapacity,omitempty"`
-	SSHPublicKey                       *string       `json:"sshPublicKey,omitempty"`
+	SSHPublicKey                       *string       `json:"sshPublicKey,omitempty" value:"required"`
 	StorageAccountDomain               *string       `json:"storageAccountDomain,omitempty" value:"required"`
 	SubscriptionResourceGroupName      *string       `json:"subscriptionResourceGroupName,omitempty" value:"required"`
 	SubscriptionResourceGroupLocation  *string       `json:"subscriptionResourceGroupLocation,omitempty" value:"required"`
@@ -151,16 +148,6 @@ func (conf *RPConfig) validate() error {
 	v := reflect.ValueOf(*configuration)
 	missingFields := []string{}
 
-	if conf.Configuration.SSHPublicKey == nil {
-		key, err := rsa.GenerateKey(rand.Reader, 2048)
-		if err != nil {
-			return err
-		}
-		pubKey := x509.MarshalPKCS1PublicKey(&key.PublicKey)
-		publicKey := string(pubKey)
-		conf.Configuration.SSHPublicKey = (&publicKey)
-	}
-
 	for i := 0; i < v.NumField(); i++ {
 		required := v.Type().Field(i).Tag.Get("value") == "required"
 
diff --git a/pkg/frontend/admin_openshiftcluster_cordonnode.go b/pkg/frontend/admin_openshiftcluster_cordonnode.go
new file mode 100644
index 000000000..be76269eb
--- /dev/null
+++ b/pkg/frontend/admin_openshiftcluster_cordonnode.go
@@ -0,0 +1,56 @@
+package frontend
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/database/cosmosdb"
+	"github.com/Azure/ARO-RP/pkg/frontend/middleware"
+)
+
+func (f *frontend) postAdminOpenShiftClusterCordonNode(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry)
+	r.URL.Path = filepath.Dir(r.URL.Path)
+
+	err := f._postAdminOpenShiftClusterCordonNode(ctx, r, log)
+
+	adminReply(log, w, nil, nil, err)
+}
+
+func (f *frontend) _postAdminOpenShiftClusterCordonNode(ctx context.Context, r *http.Request, log *logrus.Entry) error {
+	vars := mux.Vars(r)
+
+	vmName := r.URL.Query().Get("vmName")
+	shouldCordon := strings.EqualFold(r.URL.Query().Get("shouldCordon"), "true")
+	err := validateAdminKubernetesObjects(r.Method, "Node", "", vmName)
+	if err != nil {
+		return err
+	}
+
+	resourceID := strings.TrimPrefix(r.URL.Path, "/admin")
+
+	doc, err := f.dbOpenShiftClusters.Get(ctx, resourceID)
+	switch {
+	case cosmosdb.IsErrorStatusCode(err, http.StatusNotFound):
+		return api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeResourceNotFound, "", "The Resource '%s/%s' under resource group '%s' was not found.", vars["resourceType"], vars["resourceName"], vars["resourceGroupName"])
+	case err != nil:
+		return err
+	}
+
+	k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster)
+	if err != nil {
+		return err
+	}
+
+	return k.CordonNode(ctx, vmName, shouldCordon)
+}
diff --git a/pkg/frontend/admin_openshiftcluster_drainnode.go b/pkg/frontend/admin_openshiftcluster_drainnode.go
new file mode 100644
index 000000000..3ae467fab
--- /dev/null
+++ b/pkg/frontend/admin_openshiftcluster_drainnode.go
@@ -0,0 +1,55 @@
+package frontend
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/database/cosmosdb"
+	"github.com/Azure/ARO-RP/pkg/frontend/middleware"
+)
+
+func (f *frontend) postAdminOpenShiftClusterDrainNode(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	log := ctx.Value(middleware.ContextKeyLog).(*logrus.Entry)
+	r.URL.Path = filepath.Dir(r.URL.Path)
+
+	err := f._postAdminOpenShiftClusterDrainNode(ctx, r, log)
+
+	adminReply(log, w, nil, nil, err)
+}
+
+func (f *frontend) _postAdminOpenShiftClusterDrainNode(ctx context.Context, r *http.Request, log *logrus.Entry) error {
+	vars := mux.Vars(r)
+
+	vmName := r.URL.Query().Get("vmName")
+	err := validateAdminKubernetesObjects(r.Method, "Node", "", vmName)
+	if err != nil {
+		return err
+	}
+
+	resourceID := strings.TrimPrefix(r.URL.Path, "/admin")
+
+	doc, err := f.dbOpenShiftClusters.Get(ctx, resourceID)
+	switch {
+	case cosmosdb.IsErrorStatusCode(err, http.StatusNotFound):
+		return api.NewCloudError(http.StatusNotFound, api.CloudErrorCodeResourceNotFound, "", "The Resource '%s/%s' under resource group '%s' was not found.", vars["resourceType"], vars["resourceName"], vars["resourceGroupName"])
+	case err != nil:
+		return err
+	}
+
+	k, err := f.kubeActionsFactory(log, f.env, doc.OpenShiftCluster)
+	if err != nil {
+		return err
+	}
+
+	return k.DrainNode(ctx, vmName)
+}
diff --git a/pkg/frontend/adminactions/drain.go b/pkg/frontend/adminactions/drain.go
new file mode 100644
index 000000000..ab0682c22
--- /dev/null
+++ b/pkg/frontend/adminactions/drain.go
@@ -0,0 +1,59 @@
+package adminactions
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"log"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubectl/pkg/drain"
+)
+
+func (k *kubeActions) CordonNode(ctx context.Context, nodeName string, shouldCordon bool) error {
+	node, err := k.kubecli.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	drainer := &drain.Helper{
+		Ctx:                 ctx,
+		Client:              k.kubecli,
+		Force:               true,
+		GracePeriodSeconds:  -1,
+		IgnoreAllDaemonSets: true,
+		Timeout:             60 * time.Second,
+		DeleteEmptyDirData:  true,
+		DisableEviction:     true,
+		OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) {
+			log.Printf("deleted pod %s/%s", pod.Namespace, pod.Name)
+		},
+		Out:    log.Writer(),
+		ErrOut: log.Writer(),
+	}
+
+	return drain.RunCordonOrUncordon(drainer, node, shouldCordon)
+}
+
+func (k *kubeActions) DrainNode(ctx context.Context, nodeName string) error {
+	drainer := &drain.Helper{
+		Ctx:                 ctx,
+		Client:              k.kubecli,
+		Force:               true,
+		GracePeriodSeconds:  -1,
+		IgnoreAllDaemonSets: true,
+		Timeout:             3 * time.Minute,
+		DeleteEmptyDirData:  true,
+		DisableEviction:     true,
+		OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) {
+			log.Printf("deleted pod %s/%s", pod.Namespace, pod.Name)
+		},
+		Out:    log.Writer(),
+		ErrOut: log.Writer(),
+	}
+
+	return drain.RunNodeDrain(drainer, nodeName)
+}
diff --git a/pkg/frontend/adminactions/kubeactions.go b/pkg/frontend/adminactions/kubeactions.go
index 8113e416f..48511bc03 100644
--- a/pkg/frontend/adminactions/kubeactions.go
+++ b/pkg/frontend/adminactions/kubeactions.go
@@ -28,6 +28,8 @@ type KubeActions interface {
 	KubeList(ctx context.Context, groupKind, namespace string) ([]byte, error)
 	KubeCreateOrUpdate(ctx context.Context, obj *unstructured.Unstructured) error
 	KubeDelete(ctx context.Context, groupKind, namespace, name string) error
+	CordonNode(ctx context.Context, nodeName string, unschedulable bool) error
+	DrainNode(ctx context.Context, nodeName string) error
 	Upgrade(ctx context.Context, upgradeY bool) error
 	KubeGetPodLogs(ctx context.Context, namespace, name, containerName string) ([]byte, error)
 }
diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go
index cbb921fe0..82c0acec6 100644
--- a/pkg/frontend/frontend.go
+++ b/pkg/frontend/frontend.go
@@ -300,6 +300,18 @@ func (f *frontend) authenticatedRoutes(r *mux.Router) {
 
 	s.Methods(http.MethodPost).HandlerFunc(f.postAdminReconcileFailedNIC).Name("reconcileFailedNic")
 
+	s = r.
+		Path("/admin/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/cordonnode").
+		Subrouter()
+
+	s.Methods(http.MethodPost).HandlerFunc(f.postAdminOpenShiftClusterCordonNode).Name("postAdminOpenShiftClusterCordonNode")
+
+	s = r.
+		Path("/admin/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/drainnode").
+		Subrouter()
+
+	s.Methods(http.MethodPost).HandlerFunc(f.postAdminOpenShiftClusterDrainNode).Name("postAdminOpenShiftClusterDrainNode")
+
 	// Operations
 	s = r.
 		Path("/providers/{resourceProviderNamespace}/operations").
diff --git a/pkg/util/mocks/adminactions/adminactions.go b/pkg/util/mocks/adminactions/adminactions.go
index 506e7a24d..28aa65d44 100644
--- a/pkg/util/mocks/adminactions/adminactions.go
+++ b/pkg/util/mocks/adminactions/adminactions.go
@@ -38,6 +38,34 @@ func (m *MockKubeActions) EXPECT() *MockKubeActionsMockRecorder {
 	return m.recorder
 }
 
+// CordonNode mocks base method.
+func (m *MockKubeActions) CordonNode(arg0 context.Context, arg1 string, arg2 bool) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CordonNode", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CordonNode indicates an expected call of CordonNode.
+func (mr *MockKubeActionsMockRecorder) CordonNode(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CordonNode", reflect.TypeOf((*MockKubeActions)(nil).CordonNode), arg0, arg1, arg2)
+}
+
+// DrainNode mocks base method.
+func (m *MockKubeActions) DrainNode(arg0 context.Context, arg1 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DrainNode", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DrainNode indicates an expected call of DrainNode.
+func (mr *MockKubeActionsMockRecorder) DrainNode(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DrainNode", reflect.TypeOf((*MockKubeActions)(nil).DrainNode), arg0, arg1)
+}
+
 // KubeCreateOrUpdate mocks base method.
 func (m *MockKubeActions) KubeCreateOrUpdate(arg0 context.Context, arg1 *unstructured.Unstructured) error {
 	m.ctrl.T.Helper()


Test patch:
diff --git a/pkg/cluster/hive_test.go b/pkg/cluster/hive_test.go
index b93dd9d9b..a870472f1 100644
--- a/pkg/cluster/hive_test.go
+++ b/pkg/cluster/hive_test.go
@@ -6,18 +6,14 @@ package cluster
 import (
 	"context"
 	"errors"
-	"fmt"
-	"strings"
 	"testing"
 
 	"github.com/golang/mock/gomock"
 	"github.com/sirupsen/logrus"
-	corev1 "k8s.io/api/core/v1"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/hive"
 	mock_hive "github.com/Azure/ARO-RP/pkg/util/mocks/hive"
-	testdatabase "github.com/Azure/ARO-RP/test/database"
 )
 
 func TestHiveClusterDeploymentReady(t *testing.T) {
@@ -162,113 +158,3 @@ func TestHiveResetCorrelationData(t *testing.T) {
 		})
 	}
 }
-
-func TestHiveCreateNamespace(t *testing.T) {
-	for _, tt := range []struct {
-		testName              string
-		existingNamespaceName string
-		newNamespaceName      string
-		clusterManagerMock    func(mockCtrl *gomock.Controller, namespaceName string) *mock_hive.MockClusterManager
-		expectedNamespaceName string
-		wantErr               string
-	}{
-		{
-			testName:              "doesn't return error if cluster manager is nil",
-			existingNamespaceName: "",
-			newNamespaceName:      "new-namespace",
-			expectedNamespaceName: "",
-		},
-		{
-			testName:              "creates namespace if it doesn't exist",
-			existingNamespaceName: "",
-			newNamespaceName:      "new-namespace",
-			clusterManagerMock: func(mockCtrl *gomock.Controller, namespaceName string) *mock_hive.MockClusterManager {
-				namespaceToReturn := &corev1.Namespace{}
-				namespaceToReturn.Name = namespaceName
-				mockClusterManager := mock_hive.NewMockClusterManager(mockCtrl)
-				mockClusterManager.EXPECT().CreateNamespace(gomock.Any()).Return(namespaceToReturn, nil)
-				return mockClusterManager
-			},
-			expectedNamespaceName: "new-namespace",
-			wantErr:               "",
-		},
-		{
-			testName:              "doesn't create namespace if it already exists",
-			existingNamespaceName: "existing-namespace",
-			newNamespaceName:      "new-namespace",
-			expectedNamespaceName: "existing-namespace",
-			clusterManagerMock: func(mockCtrl *gomock.Controller, namespaceName string) *mock_hive.MockClusterManager {
-				mockClusterManager := mock_hive.NewMockClusterManager(mockCtrl)
-				mockClusterManager.EXPECT().CreateNamespace(gomock.Any()).Times(0)
-				return mockClusterManager
-			},
-		},
-		{
-			testName:              "returns error if cluster manager returns error",
-			existingNamespaceName: "",
-			newNamespaceName:      "new-namespace",
-			expectedNamespaceName: "",
-			clusterManagerMock: func(mockCtrl *gomock.Controller, namespaceName string) *mock_hive.MockClusterManager {
-				mockClusterManager := mock_hive.NewMockClusterManager(mockCtrl)
-				mockClusterManager.EXPECT().CreateNamespace(gomock.Any()).Return(nil, fmt.Errorf("cluster manager error"))
-				return mockClusterManager
-			},
-			wantErr: "cluster manager error",
-		},
-	} {
-		t.Run(tt.testName, func(t *testing.T) {
-			ctx := context.Background()
-			controller := gomock.NewController(t)
-			defer controller.Finish()
-
-			m := createManagerForTests(t, tt.existingNamespaceName)
-
-			if tt.clusterManagerMock != nil {
-				m.hiveClusterManager = tt.clusterManagerMock(controller, tt.newNamespaceName)
-			}
-
-			err := m.hiveCreateNamespace(ctx)
-			if err != nil && err.Error() != tt.wantErr ||
-				err == nil && tt.wantErr != "" {
-				t.Error(err)
-			}
-
-			if m.doc.OpenShiftCluster.Properties.HiveProfile.Namespace != tt.expectedNamespaceName {
-				t.Errorf("expected namespace to be %s, got %s",
-					tt.expectedNamespaceName, m.doc.OpenShiftCluster.Properties.HiveProfile.Namespace)
-			}
-		},
-		)
-	}
-}
-
-func createManagerForTests(t *testing.T, existingNamespaceName string) *manager {
-	fakeDb, _ := testdatabase.NewFakeOpenShiftClusters()
-	key := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName1"
-
-	doc := &api.OpenShiftClusterDocument{
-		Key: strings.ToLower(key),
-		OpenShiftCluster: &api.OpenShiftCluster{
-			ID: key,
-			Properties: api.OpenShiftClusterProperties{
-				HiveProfile: api.HiveProfile{
-					Namespace: existingNamespaceName,
-				},
-			},
-		},
-	}
-
-	fixture := testdatabase.NewFixture().WithOpenShiftClusters(fakeDb)
-	fixture.AddOpenShiftClusterDocuments(doc)
-	err := fixture.Create()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	m := &manager{
-		log: logrus.NewEntry(logrus.StandardLogger()),
-		db:  fakeDb,
-		doc: doc,
-	}
-	return m
-}
diff --git a/pkg/frontend/admin_openshiftcluster_cordonnode_test.go b/pkg/frontend/admin_openshiftcluster_cordonnode_test.go
new file mode 100644
index 000000000..d9902d639
--- /dev/null
+++ b/pkg/frontend/admin_openshiftcluster_cordonnode_test.go
@@ -0,0 +1,178 @@
+package frontend
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/sirupsen/logrus"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/frontend/adminactions"
+	"github.com/Azure/ARO-RP/pkg/metrics/noop"
+	mock_adminactions "github.com/Azure/ARO-RP/pkg/util/mocks/adminactions"
+	testdatabase "github.com/Azure/ARO-RP/test/database"
+)
+
+func TestAdminCordonUncordonNode(t *testing.T) {
+	mockSubID := "00000000-0000-0000-0000-000000000000"
+	mockTenantID := "00000000-0000-0000-0000-000000000000"
+
+	ctx := context.Background()
+
+	type test struct {
+		name           string
+		resourceID     string
+		fixture        func(*testdatabase.Fixture)
+		vmName         string
+		shouldCordon   string
+		mocks          func(*test, *mock_adminactions.MockKubeActions)
+		wantStatusCode int
+		wantResponse   []byte
+		wantError      string
+	}
+
+	for _, tt := range []*test{
+		{
+			name:         "basic coverage - cordon",
+			vmName:       "aro-worker-australiasoutheast-7tcq7",
+			shouldCordon: "false",
+			resourceID:   testdatabase.GetResourcePath(mockSubID, "resourceName"),
+			fixture: func(f *testdatabase.Fixture) {
+				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
+					Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					OpenShiftCluster: &api.OpenShiftCluster{
+						ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
+						Properties: api.OpenShiftClusterProperties{
+							ClusterProfile: api.ClusterProfile{
+								ResourceGroupID: fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster", mockSubID),
+							},
+						},
+					},
+				})
+
+				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
+					ID: mockSubID,
+					Subscription: &api.Subscription{
+						State: api.SubscriptionStateRegistered,
+						Properties: &api.SubscriptionProperties{
+							TenantID: mockTenantID,
+						},
+					},
+				})
+			},
+			mocks: func(tt *test, k *mock_adminactions.MockKubeActions) {
+				k.EXPECT().CordonNode(gomock.Any(), tt.vmName, false).Return(nil)
+			},
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:         "basic coverage - uncordon",
+			vmName:       "aro-worker-australiasoutheast-7tcq7",
+			shouldCordon: "true",
+			resourceID:   testdatabase.GetResourcePath(mockSubID, "resourceName"),
+			fixture: func(f *testdatabase.Fixture) {
+				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
+					Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					OpenShiftCluster: &api.OpenShiftCluster{
+						ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
+						Properties: api.OpenShiftClusterProperties{
+							ClusterProfile: api.ClusterProfile{
+								ResourceGroupID: fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster", mockSubID),
+							},
+						},
+					},
+				})
+
+				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
+					ID: mockSubID,
+					Subscription: &api.Subscription{
+						State: api.SubscriptionStateRegistered,
+						Properties: &api.SubscriptionProperties{
+							TenantID: mockTenantID,
+						},
+					},
+				})
+			},
+			mocks: func(tt *test, k *mock_adminactions.MockKubeActions) {
+				k.EXPECT().CordonNode(gomock.Any(), tt.vmName, true).Return(nil)
+			},
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:         "unable to parse schedulable parameter - default to false",
+			vmName:       "aro-worker-australiasoutheast-7tcq7",
+			shouldCordon: "unable_to_parse",
+			resourceID:   testdatabase.GetResourcePath(mockSubID, "resourceName"),
+			fixture: func(f *testdatabase.Fixture) {
+				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
+					Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					OpenShiftCluster: &api.OpenShiftCluster{
+						ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
+						Properties: api.OpenShiftClusterProperties{
+							ClusterProfile: api.ClusterProfile{
+								ResourceGroupID: fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster", mockSubID),
+							},
+						},
+					},
+				})
+
+				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
+					ID: mockSubID,
+					Subscription: &api.Subscription{
+						State: api.SubscriptionStateRegistered,
+						Properties: &api.SubscriptionProperties{
+							TenantID: mockTenantID,
+						},
+					},
+				})
+			},
+			mocks: func(tt *test, k *mock_adminactions.MockKubeActions) {
+				k.EXPECT().CordonNode(gomock.Any(), tt.vmName, false).Return(nil)
+			},
+			wantStatusCode: http.StatusOK,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions()
+			defer ti.done()
+
+			k := mock_adminactions.NewMockKubeActions(ti.controller)
+			tt.mocks(tt, k)
+
+			err := ti.buildFixtures(tt.fixture)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, api.APIs, &noop.Noop{}, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) {
+				return k, nil
+			}, nil, nil)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			go f.Run(ctx, nil, nil)
+
+			resp, b, err := ti.request(http.MethodPost,
+				fmt.Sprintf("https://server/admin%s/cordonnode?vmName=%s&shouldCordon=%s", tt.resourceID, tt.vmName, tt.shouldCordon),
+				nil, nil)
+			if err != nil {
+				t.Error(err)
+			}
+
+			err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse)
+			if err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
diff --git a/pkg/frontend/admin_openshiftcluster_drainnode_test.go b/pkg/frontend/admin_openshiftcluster_drainnode_test.go
new file mode 100644
index 000000000..47d05c8f4
--- /dev/null
+++ b/pkg/frontend/admin_openshiftcluster_drainnode_test.go
@@ -0,0 +1,110 @@
+package frontend
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/sirupsen/logrus"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/frontend/adminactions"
+	"github.com/Azure/ARO-RP/pkg/metrics/noop"
+	mock_adminactions "github.com/Azure/ARO-RP/pkg/util/mocks/adminactions"
+	testdatabase "github.com/Azure/ARO-RP/test/database"
+)
+
+func TestAdminDrainNode(t *testing.T) {
+	mockSubID := "00000000-0000-0000-0000-000000000000"
+	mockTenantID := "00000000-0000-0000-0000-000000000000"
+
+	ctx := context.Background()
+
+	type test struct {
+		name           string
+		resourceID     string
+		fixture        func(*testdatabase.Fixture)
+		vmName         string
+		mocks          func(*test, *mock_adminactions.MockKubeActions)
+		wantStatusCode int
+		wantResponse   []byte
+		wantError      string
+	}
+
+	for _, tt := range []*test{
+		{
+			name:       "basic coverage",
+			vmName:     "aro-worker-australiasoutheast-7tcq7",
+			resourceID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
+			fixture: func(f *testdatabase.Fixture) {
+				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
+					Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					OpenShiftCluster: &api.OpenShiftCluster{
+						ID: testdatabase.GetResourcePath(mockSubID, "resourceName"),
+						Properties: api.OpenShiftClusterProperties{
+							ClusterProfile: api.ClusterProfile{
+								ResourceGroupID: fmt.Sprintf("/subscriptions/%s/resourceGroups/test-cluster", mockSubID),
+							},
+						},
+					},
+				})
+
+				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
+					ID: mockSubID,
+					Subscription: &api.Subscription{
+						State: api.SubscriptionStateRegistered,
+						Properties: &api.SubscriptionProperties{
+							TenantID: mockTenantID,
+						},
+					},
+				})
+			},
+			mocks: func(tt *test, k *mock_adminactions.MockKubeActions) {
+				k.EXPECT().DrainNode(gomock.Any(), tt.vmName).Return(nil)
+			},
+			wantStatusCode: http.StatusOK,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions()
+			defer ti.done()
+
+			k := mock_adminactions.NewMockKubeActions(ti.controller)
+			tt.mocks(tt, k)
+
+			err := ti.buildFixtures(tt.fixture)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			f, err := NewFrontend(ctx, ti.audit, ti.log, ti.env, ti.asyncOperationsDatabase, ti.openShiftClustersDatabase, ti.subscriptionsDatabase, api.APIs, &noop.Noop{}, nil, func(*logrus.Entry, env.Interface, *api.OpenShiftCluster) (adminactions.KubeActions, error) {
+				return k, nil
+			}, nil, nil)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			go f.Run(ctx, nil, nil)
+
+			resp, b, err := ti.request(http.MethodPost,
+				fmt.Sprintf("https://server/admin%s/drainnode?vmName=%s", tt.resourceID, tt.vmName),
+				nil, nil)
+			if err != nil {
+				t.Error(err)
+			}
+
+			err = validateResponse(resp, b, tt.wantStatusCode, tt.wantError, tt.wantResponse)
+			if err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
diff --git a/test/e2e/adminapi_drainnode.go b/test/e2e/adminapi_drainnode.go
new file mode 100644
index 000000000..7190669ec
--- /dev/null
+++ b/test/e2e/adminapi_drainnode.go
@@ -0,0 +1,127 @@
+package e2e
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/kubectl/pkg/drain"
+)
+
+var _ = Describe("[Admin API] Cordon and Drain node actions", func() {
+	BeforeEach(skipIfNotInDevelopmentEnv)
+
+	It("should be able to cordon, drain, and uncordon nodes", func() {
+		By("selecting a worker node in the cluster")
+		nodes, err := clients.Kubernetes.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
+			LabelSelector: "node-role.kubernetes.io/worker",
+		})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(nodes.Items)).Should(BeNumerically(">", 0))
+		node := nodes.Items[0]
+		nodeName := node.Name
+
+		drainer := &drain.Helper{
+			Ctx:                 context.Background(),
+			Client:              clients.Kubernetes,
+			Force:               true,
+			GracePeriodSeconds:  -1,
+			IgnoreAllDaemonSets: true,
+			Timeout:             60 * time.Second,
+			DeleteEmptyDirData:  true,
+			DisableEviction:     true,
+			OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) {
+				log.Printf("deleted pod %s/%s", pod.Namespace, pod.Name)
+			},
+			Out:    log.Writer(),
+			ErrOut: log.Writer(),
+		}
+
+		defer func() {
+			By("uncordoning the node via Kubernetes API")
+			err = drain.RunCordonOrUncordon(drainer, &node, false)
+			Expect(err).NotTo(HaveOccurred())
+		}()
+
+		testCordonNodeOK(nodeName)
+		testDrainNodeOK(nodeName, drainer)
+		testUncordonNodeOK(nodeName)
+	})
+})
+
+func testCordonNodeOK(nodeName string) {
+	By("cordoning the node via RP admin API")
+	params := url.Values{
+		"shouldCordon": []string{"true"},
+		"vmName":       []string{nodeName},
+	}
+	resp, err := adminRequest(context.Background(), http.MethodPost, "/admin"+resourceIDFromEnv()+"/cordonnode", params, nil, nil)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(resp.StatusCode).To(Equal(http.StatusOK))
+
+	By("checking that node was cordoned via Kubernetes API")
+	node, err := clients.Kubernetes.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	Expect(node.Name).To(Equal(nodeName))
+	Expect(node.Spec.Unschedulable).Should(BeTrue())
+}
+
+func testUncordonNodeOK(nodeName string) {
+	By("uncordoning the node via RP admin API")
+	params := url.Values{
+		"shouldCordon": []string{"false"},
+		"vmName":       []string{nodeName},
+	}
+	resp, err := adminRequest(context.Background(), http.MethodPost, "/admin"+resourceIDFromEnv()+"/cordonnode", params, nil, nil)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(resp.StatusCode).To(Equal(http.StatusOK))
+
+	By("checking that node was uncordoned via Kubernetes API")
+	node, err := clients.Kubernetes.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	Expect(node.Name).To(Equal(nodeName))
+	Expect(node.Spec.Unschedulable).Should(BeFalse())
+}
+
+func testDrainNodeOK(nodeName string, drainer *drain.Helper) {
+	By("counting the number of pods on the node via Kubernetes API")
+	podsListPreDrain, err := clients.Kubernetes.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{
+		FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}).String(),
+	})
+	Expect(err).NotTo(HaveOccurred())
+	Expect(len(podsListPreDrain.Items)).Should(BeNumerically(">", 0))
+
+	By("counting the number of pods to be deleted/evicted via Kubernetes API")
+	podsForDeletion, errs := drainer.GetPodsForDeletion(nodeName)
+	Expect(errs).To(BeNil())
+	Expect(len(podsForDeletion.Pods())).Should(BeNumerically(">", 0))
+
+	By("draining the node via RP admin API")
+	params := url.Values{
+		"shouldCordon": []string{"true"},
+		"vmName":       []string{nodeName},
+	}
+	resp, err := adminRequest(context.Background(), http.MethodPost, "/admin"+resourceIDFromEnv()+"/drainnode", params, nil, nil)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(resp.StatusCode).To(Equal(http.StatusOK))
+
+	By("counting the number of pods on the drained node via Kubernetes API")
+	podsListPostDrain, err := clients.Kubernetes.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{
+		FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}).String(),
+	})
+	Expect(err).NotTo(HaveOccurred())
+	Expect(len(podsListPostDrain.Items)).Should(BeNumerically(">", 0))
+
+	By("checking that the expected number of pods exist on the drained node")
+	Expect(len(podsListPostDrain.Items)).Should(BeNumerically("<=", len(podsListPreDrain.Items)-len(podsForDeletion.Pods())))
+}