Repository: Azure/ARO-RP
Pull request #1842 (2021-11-16 20:12:51)

Revert updates to fluentbit dockerfile
### Which issue this PR addresses:

Fixes an issue attempting to build the `Dockerfile.fluentbit` that was introduced in #1731  

### What this PR does / why we need it:

Not sure why it's failing, but it is after the update.  I'm not reverting the whole PR as there is some other stuff in there that's good.  Not spending too much time investigating, but looks similar to https://github.com/rpm-software-management/microdnf/issues/27

```bash
STEP 4/6: RUN microdnf install td-agent-bit-$VERSION
Downloading metadata...

(microdnf:1): libdnf-WARNING **: 19:59:19.508: Skipping refresh of ubi-7: cannot move /var/cache/yum/metadata/ubi-7-7Server-x86_64/packages to /var/cache/yum/metadata/ubi-7-7Server-x86_64.tmp/packages
Downloading metadata...
Downloading metadata...
Downloading metadata...
Downloading metadata...
Downloading metadata...
error: Could not depsolve transaction; 1 problem detected:
 Problem: conflicting requests
  - nothing provides libpq.so.5()(64bit) needed by td-agent-bit-1.8.9-1.x86_64
  - nothing provides libsystemd.so.0()(64bit) needed by td-agent-bit-1.8.9-1.x86_64
  - nothing provides libsystemd.so.0(LIBSYSTEMD_209)(64bit) needed by td-agent-bit-1.8.9-1.x86_64
Error: error building at STEP "RUN microdnf install td-agent-bit-$VERSION": error while running runtime: exit status 1
```

### Test plan for issue:

Tested it locally

### Is there any documentation that needs to be updated for this PR?

no


Patch:
diff --git a/Dockerfile.fluentbit b/Dockerfile.fluentbit
index 95179ac6d..41c8eb8e1 100644
--- a/Dockerfile.fluentbit
+++ b/Dockerfile.fluentbit
@@ -1,8 +1,8 @@
 FROM registry.access.redhat.com/ubi7/ubi-minimal
 ARG VERSION
 RUN echo -e '[td-agent-bit]\nname=td-agent-bit\nbaseurl=https://packages.fluentbit.io/centos/7/$basearch' >/etc/yum.repos.d/td-agent-bit.repo && \
-    rpm --import https://packages.fluentbit.io/fluentbit.key
-RUN microdnf update
-RUN microdnf install td-agent-bit-$VERSION
-RUN microdnf clean all
+    rpm --import https://packages.fluentbit.io/fluentbit.key && \
+    microdnf update && \
+    microdnf install td-agent-bit-$VERSION && \
+    microdnf clean all
 ENTRYPOINT ["/opt/td-agent-bit/bin/td-agent-bit"]
diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go
index ab5b60a83..377e9cda4 100644
--- a/pkg/api/admin/openshiftcluster.go
+++ b/pkg/api/admin/openshiftcluster.go
@@ -34,6 +34,7 @@ type OpenShiftClusterProperties struct {
 	LastProvisioningState           ProvisioningState       `json:"lastProvisioningState,omitempty"`
 	FailedProvisioningState         ProvisioningState       `json:"failedProvisioningState,omitempty"`
 	LastAdminUpdateError            string                  `json:"lastAdminUpdateError,omitempty"`
+	MaintenanceTask                 MaintenanceTask         `json:"maintenanceTask,omitempty" mutable:"true"`
 	CreatedAt                       time.Time               `json:"createdAt,omitempty"`
 	CreatedBy                       string                  `json:"createdBy,omitempty"`
 	ProvisionedBy                   string                  `json:"provisionedBy,omitempty"`
@@ -75,6 +76,13 @@ const (
 	FipsValidatedModulesDisabled FipsValidatedModules = "Disabled"
 )
 
+type MaintenanceTask string
+
+const (
+	MaintenanceTaskEverything MaintenanceTask = "Everything"
+	MaintenanceTaskOperator   MaintenanceTask = "OperatorUpdate"
+)
+
 // ClusterProfile represents a cluster profile.
 type ClusterProfile struct {
 	Domain               string               `json:"domain,omitempty"`
diff --git a/pkg/api/admin/openshiftcluster_convert.go b/pkg/api/admin/openshiftcluster_convert.go
index 3229d7df1..6817b579a 100644
--- a/pkg/api/admin/openshiftcluster_convert.go
+++ b/pkg/api/admin/openshiftcluster_convert.go
@@ -25,6 +25,7 @@ func (c *openShiftClusterConverter) ToExternal(oc *api.OpenShiftCluster) interfa
 			LastProvisioningState:   ProvisioningState(oc.Properties.LastProvisioningState),
 			FailedProvisioningState: ProvisioningState(oc.Properties.FailedProvisioningState),
 			LastAdminUpdateError:    oc.Properties.LastAdminUpdateError,
+			MaintenanceTask:         MaintenanceTask(oc.Properties.MaintenanceTask),
 			CreatedAt:               oc.Properties.CreatedAt,
 			CreatedBy:               oc.Properties.CreatedBy,
 			ProvisionedBy:           oc.Properties.ProvisionedBy,
@@ -169,6 +170,7 @@ func (c *openShiftClusterConverter) ToInternal(_oc interface{}, out *api.OpenShi
 	out.Properties.LastProvisioningState = api.ProvisioningState(oc.Properties.LastProvisioningState)
 	out.Properties.FailedProvisioningState = api.ProvisioningState(oc.Properties.FailedProvisioningState)
 	out.Properties.LastAdminUpdateError = oc.Properties.LastAdminUpdateError
+	out.Properties.MaintenanceTask = api.MaintenanceTask(oc.Properties.MaintenanceTask)
 	out.Properties.CreatedBy = oc.Properties.CreatedBy
 	out.Properties.ProvisionedBy = oc.Properties.ProvisionedBy
 	out.Properties.ClusterProfile.Domain = oc.Properties.ClusterProfile.Domain
diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go
index 5d3e18f65..9911aa859 100644
--- a/pkg/api/defaults.go
+++ b/pkg/api/defaults.go
@@ -29,5 +29,12 @@ func SetDefaults(doc *OpenShiftClusterDocument) {
 		if doc.OpenShiftCluster.Properties.ClusterProfile.FipsValidatedModules == "" {
 			doc.OpenShiftCluster.Properties.ClusterProfile.FipsValidatedModules = FipsValidatedModulesDisabled
 		}
+
+		// When ProvisioningStateAdminUpdating is set, it needs a MaintenanceTask
+		if doc.OpenShiftCluster.Properties.ProvisioningState == ProvisioningStateAdminUpdating {
+			if doc.OpenShiftCluster.Properties.MaintenanceTask == "" {
+				doc.OpenShiftCluster.Properties.MaintenanceTask = MaintenanceTaskEverything
+			}
+		}
 	}
 }
diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go
index c2a1f7802..554746ace 100644
--- a/pkg/api/openshiftcluster.go
+++ b/pkg/api/openshiftcluster.go
@@ -95,6 +95,7 @@ type OpenShiftClusterProperties struct {
 	LastProvisioningState   ProvisioningState   `json:"lastProvisioningState,omitempty"`
 	FailedProvisioningState ProvisioningState   `json:"failedProvisioningState,omitempty"`
 	LastAdminUpdateError    string              `json:"lastAdminUpdateError,omitempty"`
+	MaintenanceTask         MaintenanceTask     `json:"maintenanceTask,omitempty"`
 
 	CreatedAt time.Time `json:"createdAt,omitempty"`
 
@@ -160,6 +161,13 @@ const (
 	ProvisioningStateFailed        ProvisioningState = "Failed"
 )
 
+type MaintenanceTask string
+
+const (
+	MaintenanceTaskEverything MaintenanceTask = "Everything"
+	MaintenanceTaskOperator   MaintenanceTask = "OperatorUpdate"
+)
+
 // IsTerminal returns true if state is Terminal
 func (t ProvisioningState) IsTerminal() bool {
 	return ProvisioningStateFailed == t || ProvisioningStateSucceeded == t
diff --git a/pkg/backend/openshiftcluster.go b/pkg/backend/openshiftcluster.go
index ee6ba3eb8..6445b599c 100644
--- a/pkg/backend/openshiftcluster.go
+++ b/pkg/backend/openshiftcluster.go
@@ -127,7 +127,7 @@ func (ocb *openShiftClusterBackend) handle(ctx context.Context, log *logrus.Entr
 		return ocb.endLease(ctx, log, stop, doc, api.ProvisioningStateCreating, nil)
 
 	case api.ProvisioningStateAdminUpdating:
-		log.Print("admin updating")
+		log.Printf("admin updating (type: %s)", doc.OpenShiftCluster.Properties.MaintenanceTask)
 
 		err = m.AdminUpdate(ctx)
 		if err != nil {
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index 1ae97056c..bf2ce48f2 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -29,35 +29,75 @@ import (
 
 // AdminUpdate performs an admin update of an ARO cluster
 func (m *manager) AdminUpdate(ctx context.Context) error {
-	steps := []steps.Step{
+	toRun := m.adminUpdate()
+	return m.runSteps(ctx, toRun)
+}
+
+func (m *manager) adminUpdate() []steps.Step {
+	task := m.doc.OpenShiftCluster.Properties.MaintenanceTask
+	isEverything := task == api.MaintenanceTaskEverything || task == ""
+	isOperator := task == api.MaintenanceTaskOperator
+
+	// Generic fix-up or setup actions that are fairly safe to always take, and
+	// don't require a running cluster
+	toRun := []steps.Step{
 		steps.Action(m.initializeKubernetesClients), // must be first
-		steps.Action(m.fixupClusterSPObjectID),
+		steps.Action(m.ensureBillingRecord),         // belt and braces
 		steps.Action(m.ensureDefaults),
-		steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.ensureResourceGroup)), // re-create RP RBAC if needed after tenant migration
-		steps.Action(m.createOrUpdateDenyAssignment),
+		steps.Action(m.fixupClusterSPObjectID),
+		steps.Action(m.fixInfraID), // Old clusters lacks infraID in the database. Which makes code prone to errors.
+	}
+
+	if isEverything {
+		toRun = append(toRun,
+			steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.ensureResourceGroup)), // re-create RP RBAC if needed after tenant migration
+			steps.Action(m.createOrUpdateDenyAssignment),
+			steps.Action(m.fixSSH),
+			steps.Action(m.populateDatabaseIntIP),
+			//steps.Action(m.removePrivateDNSZone), // TODO(mj): re-enable once we communicate this out
+		)
+	}
+
+	// Make sure the VMs are switched on and we have an APIServer
+	toRun = append(toRun,
 		steps.Action(m.startVMs),
-		steps.Condition(m.apiServersReady, 30*time.Minute, false),
-		steps.Action(m.populateRegistryStorageAccountName),
-		steps.Action(m.ensureBillingRecord), // belt and braces
-		steps.Action(m.configureAPIServerCertificate),
-		steps.Action(m.configureIngressCertificate),
-		steps.Action(m.fixSSH),
-		steps.Action(m.fixInfraID),        // Old clusters lacks infraID in the database. Which makes code prone to errors.
-		steps.Action(m.populateCreatedAt), // TODO(mikalai): Remove after a round of admin updates
-		steps.Action(m.fixSREKubeconfig),
-		steps.Action(m.fixUserAdminKubeconfig),
-		steps.Action(m.createOrUpdateRouterIPFromCluster),
-		steps.Action(m.populateDatabaseIntIP),
-		steps.Action(m.fixMCSCert),
-		steps.Action(m.fixMCSUserData),
-		steps.Action(m.ensureGatewayUpgrade),
-		steps.Action(m.ensureAROOperator),
-		steps.Condition(m.aroDeploymentReady, 20*time.Minute, false),
-		//steps.Action(m.removePrivateDNSZone), // TODO(mj): re-enable once we communiate this out
-		steps.Action(m.updateProvisionedBy), // Run this last so we capture the resource provider only once the upgrade has been fully performed
+		steps.Condition(m.apiServersReady, 30*time.Minute, true),
+	)
+
+	// Requires Kubernetes clients
+	if isEverything {
+		toRun = append(toRun,
+			steps.Action(m.fixSREKubeconfig),
+			steps.Action(m.fixUserAdminKubeconfig),
+			steps.Action(m.createOrUpdateRouterIPFromCluster),
+			steps.Action(m.fixMCSCert),
+			steps.Action(m.fixMCSUserData),
+			steps.Action(m.ensureGatewayUpgrade),
+			steps.Action(m.configureAPIServerCertificate),
+			steps.Action(m.configureIngressCertificate),
+			steps.Action(m.populateRegistryStorageAccountName),
+			steps.Action(m.populateCreatedAt), // TODO(mikalai): Remove after a round of admin updates
+
+		)
 	}
 
-	return m.runSteps(ctx, steps)
+	// Update the ARO Operator
+	if isEverything || isOperator {
+		toRun = append(toRun,
+			steps.Action(m.ensureAROOperator),
+			steps.Condition(m.aroDeploymentReady, 20*time.Minute, true),
+		)
+	}
+
+	// We don't run this on an operator-only deploy as PUCM scripts then cannot
+	// determine if the cluster has been fully admin-updated
+	if isEverything {
+		toRun = append(toRun,
+			steps.Action(m.updateProvisionedBy), // Run this last so we capture the resource provider only once the upgrade has been fully performed
+		)
+	}
+
+	return toRun
 }
 
 func (m *manager) Update(ctx context.Context) error {
diff --git a/pkg/database/openshiftclusters.go b/pkg/database/openshiftclusters.go
index 7db2be3f8..c91371a17 100644
--- a/pkg/database/openshiftclusters.go
+++ b/pkg/database/openshiftclusters.go
@@ -306,6 +306,7 @@ func (c *openShiftClusters) EndLease(ctx context.Context, key string, provisioni
 	return c.patchWithLease(ctx, key, func(doc *api.OpenShiftClusterDocument) error {
 		doc.OpenShiftCluster.Properties.ProvisioningState = provisioningState
 		doc.OpenShiftCluster.Properties.FailedProvisioningState = failedProvisioningState
+		doc.OpenShiftCluster.Properties.MaintenanceTask = ""
 
 		doc.LeaseOwner = ""
 		doc.LeaseExpires = 0
diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go
index c7d1d597c..087770e6d 100644
--- a/pkg/frontend/openshiftcluster_putorpatch.go
+++ b/pkg/frontend/openshiftcluster_putorpatch.go
@@ -172,9 +172,6 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.
 	// is not provided in the header must be preserved
 	f.systemDataEnricher(doc, systemData)
 
-	// SetDefaults will set defaults on cluster document
-	api.SetDefaults(doc)
-
 	if isCreate {
 		// on create, make the cluster resourcegroup ID lower case to work
 		// around LB/PLS bug
@@ -203,6 +200,9 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus.
 		doc.Dequeues = 0
 	}
 
+	// SetDefaults will set defaults on cluster document
+	api.SetDefaults(doc)
+
 	doc.AsyncOperationID, err = f.newAsyncOperation(ctx, r, doc)
 	if err != nil {
 		return nil, err


Test patch:
diff --git a/pkg/api/admin/openshiftcluster_validatestatic.go b/pkg/api/admin/openshiftcluster_validatestatic.go
index 51c23404b..f94c6a338 100644
--- a/pkg/api/admin/openshiftcluster_validatestatic.go
+++ b/pkg/api/admin/openshiftcluster_validatestatic.go
@@ -29,6 +29,10 @@ func (sv *openShiftClusterStaticValidator) validateDelta(oc, current *OpenShiftC
 		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodePropertyChangeNotAllowed, err.Target, err.Message)
 	}
 
+	if !(oc.Properties.MaintenanceTask == "" || oc.Properties.MaintenanceTask == MaintenanceTaskEverything || oc.Properties.MaintenanceTask == MaintenanceTaskOperator) {
+		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.maintenanceTask", "Invalid enum parameter.")
+	}
+
 	if current.Properties.FeatureProfile.GatewayEnabled && !oc.Properties.FeatureProfile.GatewayEnabled {
 		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodePropertyChangeNotAllowed, "properties.featureProfile.gatewayEnabled", "Changing property 'properties.featureProfile.gatewayEnabled' is not allowed.")
 	}
diff --git a/pkg/api/admin/openshiftcluster_validatestatic_test.go b/pkg/api/admin/openshiftcluster_validatestatic_test.go
index 2aca0c03f..a04448654 100644
--- a/pkg/api/admin/openshiftcluster_validatestatic_test.go
+++ b/pkg/api/admin/openshiftcluster_validatestatic_test.go
@@ -633,6 +633,59 @@ func TestOpenShiftClusterStaticValidateDelta(t *testing.T) {
 			},
 			wantErr: "400: PropertyChangeNotAllowed: properties.registryProfiles: Changing property 'properties.registryProfiles' is not allowed.",
 		},
+		{
+			name: "maintenanceTask change to Everything is allowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceTask: "",
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceTask = MaintenanceTaskEverything
+			},
+		},
+		{
+			name: "maintenanceTask change to Operator is allowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceTask: "",
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceTask = MaintenanceTaskOperator
+			},
+		},
+		{
+			name: "maintenanceTask change to blank allowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceTask: MaintenanceTaskEverything,
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceTask = ""
+			},
+		},
+		{
+			name: "maintenanceTask change to other values is disallowed",
+			oc: func() *OpenShiftCluster {
+				return &OpenShiftCluster{
+					Properties: OpenShiftClusterProperties{
+						MaintenanceTask: "",
+					},
+				}
+			},
+			modify: func(oc *OpenShiftCluster) {
+				oc.Properties.MaintenanceTask = "abababa"
+			},
+			wantErr: "400: InvalidParameter: properties.maintenanceTask: Invalid enum parameter.",
+		},
 	}
 
 	for _, tt := range tests {
diff --git a/pkg/backend/openshiftcluster_test.go b/pkg/backend/openshiftcluster_test.go
index c3105641e..4baa146a4 100644
--- a/pkg/backend/openshiftcluster_test.go
+++ b/pkg/backend/openshiftcluster_test.go
@@ -167,7 +167,7 @@ func TestBackendTry(t *testing.T) {
 			},
 		},
 		{
-			name: "StateAdminUpdating success sets the last ProvisioningState and clears LastAdminUpdateError",
+			name: "StateAdminUpdating success sets the last ProvisioningState and clears LastAdminUpdateError and MaintenanceTask",
 			fixture: func(f *testdatabase.Fixture) {
 				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
 					Key: strings.ToLower(resourceID),
@@ -180,6 +180,7 @@ func TestBackendTry(t *testing.T) {
 							ProvisioningState:     api.ProvisioningStateAdminUpdating,
 							LastProvisioningState: api.ProvisioningStateSucceeded,
 							LastAdminUpdateError:  "oh no",
+							MaintenanceTask:       api.MaintenanceTaskEverything,
 						},
 					},
 				})
@@ -219,6 +220,7 @@ func TestBackendTry(t *testing.T) {
 							ProvisioningState:       api.ProvisioningStateAdminUpdating,
 							LastProvisioningState:   api.ProvisioningStateSucceeded,
 							FailedProvisioningState: api.ProvisioningStateUpdating,
+							MaintenanceTask:         api.MaintenanceTaskEverything,
 						},
 					},
 				})
diff --git a/pkg/cluster/adminupdate_test.go b/pkg/cluster/adminupdate_test.go
new file mode 100644
index 000000000..ea2d824bf
--- /dev/null
+++ b/pkg/cluster/adminupdate_test.go
@@ -0,0 +1,144 @@
+package cluster
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/go-test/deep"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+)
+
+func TestAdminUpdateSteps(t *testing.T) {
+	const (
+		key = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName1"
+	)
+
+	baseClusterDoc := func() *api.OpenShiftClusterDocument {
+		return &api.OpenShiftClusterDocument{
+			Key: strings.ToLower(key),
+			OpenShiftCluster: &api.OpenShiftCluster{
+				ID: key,
+			},
+		}
+	}
+
+	for _, tt := range []struct {
+		name           string
+		fixture        func() *api.OpenShiftClusterDocument
+		shouldRunSteps []string
+	}{
+		{
+			name: "ARO Operator Update",
+			fixture: func() *api.OpenShiftClusterDocument {
+				doc := baseClusterDoc()
+				doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateAdminUpdating
+				doc.OpenShiftCluster.Properties.MaintenanceTask = api.MaintenanceTaskOperator
+				return doc
+			},
+			shouldRunSteps: []string{
+				"[Action initializeKubernetesClients-fm]",
+				"[Action ensureBillingRecord-fm]",
+				"[Action ensureDefaults-fm]",
+				"[Action fixupClusterSPObjectID-fm]",
+				"[Action fixInfraID-fm]",
+				"[Action startVMs-fm]",
+				"[Condition apiServersReady-fm, timeout 30m0s]",
+				"[Action ensureAROOperator-fm]",
+				"[Condition aroDeploymentReady-fm, timeout 20m0s]",
+			},
+		},
+		{
+			name: "Everything update",
+			fixture: func() *api.OpenShiftClusterDocument {
+				doc := baseClusterDoc()
+				doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateAdminUpdating
+				doc.OpenShiftCluster.Properties.MaintenanceTask = api.MaintenanceTaskEverything
+				return doc
+			},
+			shouldRunSteps: []string{
+				"[Action initializeKubernetesClients-fm]",
+				"[Action ensureBillingRecord-fm]",
+				"[Action ensureDefaults-fm]",
+				"[Action fixupClusterSPObjectID-fm]",
+				"[Action fixInfraID-fm]",
+				"[AuthorizationRefreshingAction [Action ensureResourceGroup-fm]]",
+				"[Action createOrUpdateDenyAssignment-fm]",
+				"[Action fixSSH-fm]",
+				"[Action populateDatabaseIntIP-fm]",
+				"[Action startVMs-fm]",
+				"[Condition apiServersReady-fm, timeout 30m0s]",
+				"[Action fixSREKubeconfig-fm]",
+				"[Action fixUserAdminKubeconfig-fm]",
+				"[Action createOrUpdateRouterIPFromCluster-fm]",
+				"[Action fixMCSCert-fm]",
+				"[Action fixMCSUserData-fm]",
+				"[Action ensureGatewayUpgrade-fm]",
+				"[Action configureAPIServerCertificate-fm]",
+				"[Action configureIngressCertificate-fm]",
+				"[Action populateRegistryStorageAccountName-fm]",
+				"[Action populateCreatedAt-fm]",
+				"[Action ensureAROOperator-fm]",
+				"[Condition aroDeploymentReady-fm, timeout 20m0s]",
+				"[Action updateProvisionedBy-fm]",
+			},
+		},
+		{
+			name: "Blank (should perform everything)",
+			fixture: func() *api.OpenShiftClusterDocument {
+				doc := baseClusterDoc()
+				doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateAdminUpdating
+				doc.OpenShiftCluster.Properties.MaintenanceTask = api.MaintenanceTaskEverything
+				return doc
+			},
+			shouldRunSteps: []string{
+				"[Action initializeKubernetesClients-fm]",
+				"[Action ensureBillingRecord-fm]",
+				"[Action ensureDefaults-fm]",
+				"[Action fixupClusterSPObjectID-fm]",
+				"[Action fixInfraID-fm]",
+				"[AuthorizationRefreshingAction [Action ensureResourceGroup-fm]]",
+				"[Action createOrUpdateDenyAssignment-fm]",
+				"[Action fixSSH-fm]",
+				"[Action populateDatabaseIntIP-fm]",
+				"[Action startVMs-fm]",
+				"[Condition apiServersReady-fm, timeout 30m0s]",
+				"[Action fixSREKubeconfig-fm]",
+				"[Action fixUserAdminKubeconfig-fm]",
+				"[Action createOrUpdateRouterIPFromCluster-fm]",
+				"[Action fixMCSCert-fm]",
+				"[Action fixMCSUserData-fm]",
+				"[Action ensureGatewayUpgrade-fm]",
+				"[Action configureAPIServerCertificate-fm]",
+				"[Action configureIngressCertificate-fm]",
+				"[Action populateRegistryStorageAccountName-fm]",
+				"[Action populateCreatedAt-fm]",
+				"[Action ensureAROOperator-fm]",
+				"[Condition aroDeploymentReady-fm, timeout 20m0s]",
+				"[Action updateProvisionedBy-fm]",
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			m := &manager{
+				doc: tt.fixture(),
+			}
+			toRun := m.adminUpdate()
+
+			var stepsToRun []string
+			for _, s := range toRun {
+				// make it a little nicer when defining the steps that should run, since they're all methods
+				o := strings.Replace(s.String(), "github.com/Azure/ARO-RP/pkg/cluster.(*manager).", "", -1)
+				stepsToRun = append(stepsToRun, o)
+			}
+
+			diff := deep.Equal(stepsToRun, tt.shouldRunSteps)
+			for _, d := range diff {
+				t.Error(d)
+			}
+		})
+	}
+}
diff --git a/pkg/frontend/openshiftcluster_putorpatch_test.go b/pkg/frontend/openshiftcluster_putorpatch_test.go
index 6d7a61777..72bbc2613 100644
--- a/pkg/frontend/openshiftcluster_putorpatch_test.go
+++ b/pkg/frontend/openshiftcluster_putorpatch_test.go
@@ -107,6 +107,7 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
 							ClusterProfile: api.ClusterProfile{
 								FipsValidatedModules: api.FipsValidatedModulesDisabled,
 							},
+							MaintenanceTask: api.MaintenanceTaskEverything,
 							NetworkProfile: api.NetworkProfile{
 								SoftwareDefinedNetwork: api.SoftwareDefinedNetworkOpenShiftSDN,
 							},
@@ -129,6 +130,168 @@ func TestPutOrPatchOpenShiftClusterAdminAPI(t *testing.T) {
 					ClusterProfile: admin.ClusterProfile{
 						FipsValidatedModules: admin.FipsValidatedModulesDisabled,
 					},
+					MaintenanceTask: admin.MaintenanceTaskEverything,
+					NetworkProfile: admin.NetworkProfile{
+						SoftwareDefinedNetwork: admin.SoftwareDefinedNetworkOpenShiftSDN,
+					},
+					MasterProfile: admin.MasterProfile{
+						EncryptionAtHost: admin.EncryptionAtHostDisabled,
+					},
+				},
+			},
+		},
+		{
+			name: "patch with operator update request",
+			request: func(oc *admin.OpenShiftCluster) {
+				oc.Properties.MaintenanceTask = admin.MaintenanceTaskOperator
+			},
+			isPatch: true,
+			fixture: func(f *testdatabase.Fixture) {
+				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
+					ID: mockSubID,
+					Subscription: &api.Subscription{
+						State: api.SubscriptionStateRegistered,
+					},
+				})
+				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
+					Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					OpenShiftCluster: &api.OpenShiftCluster{
+						ID:   testdatabase.GetResourcePath(mockSubID, "resourceName"),
+						Type: "Microsoft.RedHatOpenShift/openShiftClusters",
+						Tags: map[string]string{"tag": "will-be-kept"},
+						Properties: api.OpenShiftClusterProperties{
+							ProvisioningState: api.ProvisioningStateSucceeded,
+						},
+					},
+				})
+			},
+			wantSystemDataEnriched: true,
+			wantEnriched:           []string{testdatabase.GetResourcePath(mockSubID, "resourceName")},
+			wantDocuments: func(c *testdatabase.Checker) {
+				c.AddAsyncOperationDocuments(&api.AsyncOperationDocument{
+					OpenShiftClusterKey: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					AsyncOperation: &api.AsyncOperation{
+						InitialProvisioningState: api.ProvisioningStateAdminUpdating,
+						ProvisioningState:        api.ProvisioningStateAdminUpdating,
+					},
+				})
+				c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
+					Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					OpenShiftCluster: &api.OpenShiftCluster{
+						ID:   testdatabase.GetResourcePath(mockSubID, "resourceName"),
+						Type: "Microsoft.RedHatOpenShift/openShiftClusters",
+						Tags: map[string]string{"tag": "will-be-kept"},
+						Properties: api.OpenShiftClusterProperties{
+							ProvisioningState:     api.ProvisioningStateAdminUpdating,
+							LastProvisioningState: api.ProvisioningStateSucceeded,
+							ClusterProfile: api.ClusterProfile{
+								FipsValidatedModules: api.FipsValidatedModulesDisabled,
+							},
+							MaintenanceTask: api.MaintenanceTaskOperator,
+							NetworkProfile: api.NetworkProfile{
+								SoftwareDefinedNetwork: api.SoftwareDefinedNetworkOpenShiftSDN,
+							},
+							MasterProfile: api.MasterProfile{
+								EncryptionAtHost: api.EncryptionAtHostDisabled,
+							},
+						},
+					},
+				})
+			},
+			wantAsync:      true,
+			wantStatusCode: http.StatusOK,
+			wantResponse: &admin.OpenShiftCluster{
+				ID:   testdatabase.GetResourcePath(mockSubID, "resourceName"),
+				Type: "Microsoft.RedHatOpenShift/openShiftClusters",
+				Tags: map[string]string{"tag": "will-be-kept"},
+				Properties: admin.OpenShiftClusterProperties{
+					ProvisioningState:     admin.ProvisioningStateAdminUpdating,
+					LastProvisioningState: admin.ProvisioningStateSucceeded,
+					ClusterProfile: admin.ClusterProfile{
+						FipsValidatedModules: admin.FipsValidatedModulesDisabled,
+					},
+					MaintenanceTask: admin.MaintenanceTaskOperator,
+					NetworkProfile: admin.NetworkProfile{
+						SoftwareDefinedNetwork: admin.SoftwareDefinedNetworkOpenShiftSDN,
+					},
+					MasterProfile: admin.MasterProfile{
+						EncryptionAtHost: admin.EncryptionAtHostDisabled,
+					},
+				},
+			},
+		},
+		{
+			name: "patch with operator update request -- existing maintenance task",
+			request: func(oc *admin.OpenShiftCluster) {
+				oc.Properties.MaintenanceTask = admin.MaintenanceTaskOperator
+			},
+			isPatch: true,
+			fixture: func(f *testdatabase.Fixture) {
+				f.AddSubscriptionDocuments(&api.SubscriptionDocument{
+					ID: mockSubID,
+					Subscription: &api.Subscription{
+						State: api.SubscriptionStateRegistered,
+					},
+				})
+				f.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
+					Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					OpenShiftCluster: &api.OpenShiftCluster{
+						ID:   testdatabase.GetResourcePath(mockSubID, "resourceName"),
+						Type: "Microsoft.RedHatOpenShift/openShiftClusters",
+						Tags: map[string]string{"tag": "will-be-kept"},
+						Properties: api.OpenShiftClusterProperties{
+							ProvisioningState: api.ProvisioningStateSucceeded,
+							MaintenanceTask:   api.MaintenanceTaskEverything,
+						},
+					},
+				})
+			},
+			wantSystemDataEnriched: true,
+			wantEnriched:           []string{testdatabase.GetResourcePath(mockSubID, "resourceName")},
+			wantDocuments: func(c *testdatabase.Checker) {
+				c.AddAsyncOperationDocuments(&api.AsyncOperationDocument{
+					OpenShiftClusterKey: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					AsyncOperation: &api.AsyncOperation{
+						InitialProvisioningState: api.ProvisioningStateAdminUpdating,
+						ProvisioningState:        api.ProvisioningStateAdminUpdating,
+					},
+				})
+				c.AddOpenShiftClusterDocuments(&api.OpenShiftClusterDocument{
+					Key: strings.ToLower(testdatabase.GetResourcePath(mockSubID, "resourceName")),
+					OpenShiftCluster: &api.OpenShiftCluster{
+						ID:   testdatabase.GetResourcePath(mockSubID, "resourceName"),
+						Type: "Microsoft.RedHatOpenShift/openShiftClusters",
+						Tags: map[string]string{"tag": "will-be-kept"},
+						Properties: api.OpenShiftClusterProperties{
+							ProvisioningState:     api.ProvisioningStateAdminUpdating,
+							LastProvisioningState: api.ProvisioningStateSucceeded,
+							ClusterProfile: api.ClusterProfile{
+								FipsValidatedModules: api.FipsValidatedModulesDisabled,
+							},
+							MaintenanceTask: api.MaintenanceTaskOperator,
+							NetworkProfile: api.NetworkProfile{
+								SoftwareDefinedNetwork: api.SoftwareDefinedNetworkOpenShiftSDN,
+							},
+							MasterProfile: api.MasterProfile{
+								EncryptionAtHost: api.EncryptionAtHostDisabled,
+							},
+						},
+					},
+				})
+			},
+			wantAsync:      true,
+			wantStatusCode: http.StatusOK,
+			wantResponse: &admin.OpenShiftCluster{
+				ID:   testdatabase.GetResourcePath(mockSubID, "resourceName"),
+				Type: "Microsoft.RedHatOpenShift/openShiftClusters",
+				Tags: map[string]string{"tag": "will-be-kept"},
+				Properties: admin.OpenShiftClusterProperties{
+					ProvisioningState:     admin.ProvisioningStateAdminUpdating,
+					LastProvisioningState: admin.ProvisioningStateSucceeded,
+					ClusterProfile: admin.ClusterProfile{
+						FipsValidatedModules: admin.FipsValidatedModulesDisabled,
+					},
+					MaintenanceTask: admin.MaintenanceTaskOperator,
 					NetworkProfile: admin.NetworkProfile{
 						SoftwareDefinedNetwork: admin.SoftwareDefinedNetworkOpenShiftSDN,
 					},