Repository: Azure/ARO-RP
Pull request #2399 (2022-09-16 04:26:35)

Certificate renewal PUCM type to rotate in-cluster MDSD/ingress/API certs
### Which issue this PR addresses:

Fixes: https://msazure.visualstudio.com/AzureRedHatOpenShift/_workitems/edit/15581636/

### What this PR does / why we need it:

During regular PUCM few of the clusters are failing and these clusters are unable to fetch the new certificates from RP as these steps never run after the PUCM failure due to customer policies or other issues. With this PR we can just simply run the steps which will fetch new certificates and install them in the cluster without the need to run all the steps in regular PUCM.

### Test plan for issue:

No Test Cases for this issue. Just re-using the existing functions which already have test cases written.

CURL Command to perform Admin Update just for certificates renewal:
curl -X PATCH -k "https://localhost:8443/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCEGROUP/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER?api-version=admin" --header "Content-Type: application/json" -d '{"properties":{"maintenanceTask": "CertificatesRenewal"}}'


### Is there any documentation that needs to be updated for this PR?





Patch:
diff --git a/pkg/api/admin/openshiftcluster.go b/pkg/api/admin/openshiftcluster.go
index ad8b79fdb..c23e10965 100644
--- a/pkg/api/admin/openshiftcluster.go
+++ b/pkg/api/admin/openshiftcluster.go
@@ -84,6 +84,7 @@ type MaintenanceTask string
 const (
 	MaintenanceTaskEverything MaintenanceTask = "Everything"
 	MaintenanceTaskOperator   MaintenanceTask = "OperatorUpdate"
+	MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
 )
 
 // Operator feature flags
diff --git a/pkg/api/openshiftcluster.go b/pkg/api/openshiftcluster.go
index dac969e6e..8d787cfe0 100644
--- a/pkg/api/openshiftcluster.go
+++ b/pkg/api/openshiftcluster.go
@@ -173,6 +173,7 @@ type MaintenanceTask string
 const (
 	MaintenanceTaskEverything MaintenanceTask = "Everything"
 	MaintenanceTaskOperator   MaintenanceTask = "OperatorUpdate"
+	MaintenanceTaskRenewCerts MaintenanceTask = "CertificatesRenewal"
 )
 
 // Cluster-scoped flags
diff --git a/pkg/cluster/arooperator.go b/pkg/cluster/arooperator.go
index 638947516..3b85ee7c6 100644
--- a/pkg/cluster/arooperator.go
+++ b/pkg/cluster/arooperator.go
@@ -49,3 +49,7 @@ func (m *manager) ensureAROOperatorRunningDesiredVersion(ctx context.Context) (b
 	}
 	return true, nil
 }
+
+func (m *manager) renewMDSDCertificate(ctx context.Context) error {
+	return m.aroOperatorDeployer.RenewMDSDCertificate(ctx)
+}
diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go
index c908d8bdc..f9fe872d1 100644
--- a/pkg/cluster/install.go
+++ b/pkg/cluster/install.go
@@ -37,6 +37,7 @@ func (m *manager) adminUpdate() []steps.Step {
 	task := m.doc.OpenShiftCluster.Properties.MaintenanceTask
 	isEverything := task == api.MaintenanceTaskEverything || task == ""
 	isOperator := task == api.MaintenanceTaskOperator
+	isRenewCerts := task == api.MaintenanceTaskRenewCerts
 
 	// Generic fix-up or setup actions that are fairly safe to always take, and
 	// don't require a running cluster
@@ -73,20 +74,50 @@ func (m *manager) adminUpdate() []steps.Step {
 			steps.Action(m.fixSREKubeconfig),
 			steps.Action(m.fixUserAdminKubeconfig),
 			steps.Action(m.createOrUpdateRouterIPFromCluster),
+		)
+	}
+
+	if isEverything || isRenewCerts {
+		toRun = append(toRun,
 			steps.Action(m.fixMCSCert),
 			steps.Action(m.fixMCSUserData),
+		)
+	}
+
+	if isEverything {
+		toRun = append(toRun,
 			steps.Action(m.ensureGatewayUpgrade),
+		)
+	}
+
+	if isEverything || isRenewCerts {
+		toRun = append(toRun,
 			steps.Action(m.configureAPIServerCertificate),
 			steps.Action(m.configureIngressCertificate),
+		)
+	}
+
+	if isEverything {
+		toRun = append(toRun,
 			steps.Action(m.populateRegistryStorageAccountName),
 			steps.Action(m.ensureMTUSize),
 		)
 	}
 
+	if isEverything || isOperator || isRenewCerts {
+		toRun = append(toRun,
+			steps.Action(m.initializeOperatorDeployer))
+	}
+
+	if isRenewCerts {
+		toRun = append(toRun,
+			steps.Action(m.renewMDSDCertificate),
+		)
+	}
+
 	// Update the ARO Operator
 	if isEverything || isOperator {
 		toRun = append(toRun,
-			steps.Action(m.initializeOperatorDeployer), // depends on kube clients
 			steps.Action(m.ensureAROOperator),
 			steps.Condition(m.aroDeploymentReady, 20*time.Minute, true),
 			steps.Condition(m.ensureAROOperatorRunningDesiredVersion, 5*time.Minute, true),
diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go
index 5e0dc7069..2ce418004 100644
--- a/pkg/operator/deploy/deploy.go
+++ b/pkg/operator/deploy/deploy.go
@@ -50,6 +50,7 @@ type Operator interface {
 	CreateOrUpdate(context.Context) error
 	IsReady(context.Context) (bool, error)
 	IsRunningDesiredVersion(context.Context) (bool, error)
+	RenewMDSDCertificate(context.Context) error
 }
 
 type operator struct {
@@ -364,6 +365,31 @@ func (o *operator) CreateOrUpdate(ctx context.Context) error {
 	return nil
 }
 
+func (o *operator) RenewMDSDCertificate(ctx context.Context) error {
+	key, cert := o.env.ClusterGenevaLoggingSecret()
+	gcsKeyBytes, err := utiltls.PrivateKeyAsBytes(key)
+	if err != nil {
+		return err
+	}
+	gcsCertBytes, err := utiltls.CertAsBytes(cert)
+	if err != nil {
+		return err
+	}
+
+	s, err := o.kubernetescli.CoreV1().Secrets(pkgoperator.Namespace).Get(ctx, pkgoperator.SecretName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	s.Data["gcscert.pem"] = gcsCertBytes
+	s.Data["gcskey.pem"] = gcsKeyBytes
+
+	_, err = o.kubernetescli.CoreV1().Secrets(pkgoperator.Namespace).Update(ctx, s, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (o *operator) IsReady(ctx context.Context) (bool, error) {
 	ok, err := ready.CheckDeploymentIsReady(ctx, o.kubernetescli.AppsV1().Deployments(pkgoperator.Namespace), "aro-operator-master")()
 	if !ok || err != nil {
diff --git a/pkg/util/mocks/operator/deploy/deploy.go b/pkg/util/mocks/operator/deploy/deploy.go
index 5d5009d70..4b42b5609 100644
--- a/pkg/util/mocks/operator/deploy/deploy.go
+++ b/pkg/util/mocks/operator/deploy/deploy.go
@@ -77,3 +77,17 @@ func (mr *MockOperatorMockRecorder) IsRunningDesiredVersion(arg0 interface{}) *g
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsRunningDesiredVersion", reflect.TypeOf((*MockOperator)(nil).IsRunningDesiredVersion), arg0)
 }
+
+// RenewMDSDCertificate mocks base method.
+func (m *MockOperator) RenewMDSDCertificate(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RenewMDSDCertificate", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RenewMDSDCertificate indicates an expected call of RenewMDSDCertificate.
+func (mr *MockOperatorMockRecorder) RenewMDSDCertificate(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenewMDSDCertificate", reflect.TypeOf((*MockOperator)(nil).RenewMDSDCertificate), arg0)
+}


Test patch:
diff --git a/pkg/api/admin/openshiftcluster_validatestatic.go b/pkg/api/admin/openshiftcluster_validatestatic.go
index c4346edf3..cad10cbf2 100644
--- a/pkg/api/admin/openshiftcluster_validatestatic.go
+++ b/pkg/api/admin/openshiftcluster_validatestatic.go
@@ -29,7 +29,7 @@ func (sv openShiftClusterStaticValidator) validateDelta(oc, current *OpenShiftCl
 		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodePropertyChangeNotAllowed, err.Target, err.Message)
 	}
 
-	if !(oc.Properties.MaintenanceTask == "" || oc.Properties.MaintenanceTask == MaintenanceTaskEverything || oc.Properties.MaintenanceTask == MaintenanceTaskOperator) {
+	if !(oc.Properties.MaintenanceTask == "" || oc.Properties.MaintenanceTask == MaintenanceTaskEverything || oc.Properties.MaintenanceTask == MaintenanceTaskOperator || oc.Properties.MaintenanceTask == MaintenanceTaskRenewCerts) {
 		return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.maintenanceTask", "Invalid enum parameter.")
 	}
 
diff --git a/pkg/cluster/adminupdate_test.go b/pkg/cluster/adminupdate_test.go
index f8a5d5155..8f8900a3f 100644
--- a/pkg/cluster/adminupdate_test.go
+++ b/pkg/cluster/adminupdate_test.go
@@ -181,6 +181,30 @@ func TestAdminUpdateSteps(t *testing.T) {
 				"[Action updateProvisionedBy-fm]",
 			},
 		},
+		{
+			name: "Rotate in-cluster MDSD/Ingress/API certs",
+			fixture: func() (*api.OpenShiftClusterDocument, bool) {
+				doc := baseClusterDoc()
+				doc.OpenShiftCluster.Properties.ProvisioningState = api.ProvisioningStateAdminUpdating
+				doc.OpenShiftCluster.Properties.MaintenanceTask = api.MaintenanceTaskRenewCerts
+				return doc, true
+			},
+			shouldRunSteps: []string{
+				"[Action initializeKubernetesClients-fm]",
+				"[Action ensureBillingRecord-fm]",
+				"[Action ensureDefaults-fm]",
+				"[Action fixupClusterSPObjectID-fm]",
+				"[Action fixInfraID-fm]",
+				"[Action startVMs-fm]",
+				"[Condition apiServersReady-fm, timeout 30m0s]",
+				"[Action fixMCSCert-fm]",
+				"[Action fixMCSUserData-fm]",
+				"[Action configureAPIServerCertificate-fm]",
+				"[Action configureIngressCertificate-fm]",
+				"[Action initializeOperatorDeployer-fm]",
+				"[Action renewMDSDCertificate-fm]",
+			},
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			doc, adoptViaHive := tt.fixture()