Repository: rook/kubectl-rook-ceph
Pull request #368: mon: add support for restoring arbitrary CRs while the Rook resources are being deleted
Author: login: OdedViner
Created: 2025-04-17T14:26:11Z | Merged: 2025-04-28T05:02:51Z

<!-- Thank you for contributing to Rook! -->
This PR extends the `RestoreCrd` functionality to support restoring additional third-party CRs 

- Adds a `CustomResource` struct to pass a list of CRs to clean up ownerReferences
- Enhances `removeOwnerRefOfUID` to work with dynamically provided CRs
- Makes the operator name configurable to support tearing down and restarting a different operator (not just rook-ceph-operator)

This change improves flexibility in disaster recovery scenarios 
<!-- STEPS TO FOLLOW:
  1. Add a description of the changes (frequently the same as the commit description)
  2. Enter the issue number next to "Resolves #" below (if there is no tracking issue resolved, **remove that section**)
  3. Review our Contributing documentation at https://rook.io/docs/rook/latest/Contributing/development-flow/
  4. Follow the steps in the checklist below, starting with the **Commit Message Formatting**.
-->

<!-- Uncomment this section with the issue number if an issue is being resolved
**Issue resolved by this Pull Request:**
Resolves #
--->

**Checklist:**

- [ ] **Commit Message Formatting**: Commit titles and messages follow guidelines in the [developer guide](https://rook.io/docs/rook/latest/Contributing/development-flow/#commit-structure).
- [ ] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/latest/Contributing/development-flow/#submitting-a-pull-request)
- [ ] Documentation has been updated, if necessary.
- [ ] Unit tests have been added, if necessary.
- [ ] Integration tests have been added, if necessary.



Diff:
diff --git a/cmd/commands/restore.go b/cmd/commands/restore.go
index e86980ad..3fe70368 100644
--- a/cmd/commands/restore.go
+++ b/cmd/commands/restore.go
@@ -32,6 +32,7 @@ var RestoreCmd = &cobra.Command{
 		verifyOperatorPodIsRunning(cmd.Context(), clientSets)
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		restore.RestoreCrd(cmd.Context(), clientSets, operatorNamespace, cephClusterNamespace, crds.CephRookIoGroup, crds.CephRookResourcesVersion, args)
+		customResources := []restore.CustomResource{}
+		restore.RestoreCrd(cmd.Context(), clientSets, operatorNamespace, cephClusterNamespace, crds.CephRookIoGroup, crds.CephRookResourcesVersion, "rook-ceph-operator", customResources, args)
 	},
 }
diff --git a/pkg/restore/crd.go b/pkg/restore/crd.go
index bc073b23..b812ecbf 100644
--- a/pkg/restore/crd.go
+++ b/pkg/restore/crd.go
@@ -29,11 +29,18 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 )
 
+type CustomResource struct {
+	Group    string
+	Version  string
+	Resource string
+}
+
 func RestoreCrd(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorNamespace, clusterNamespace,
-	groupName, versionResource string, args []string) {
+	groupName, versionResource, operatorName string, customResources []CustomResource, args []string) {
 	crd := args[0]
 
 	var crName string
@@ -76,9 +83,9 @@ func RestoreCrd(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorN
 	logging.Info("Proceeding with restoring deleting CR")
 
 	logging.Info("Scaling down the operator")
-	err = k8sutil.SetDeploymentScale(ctx, k8sclientset.Kube, operatorNamespace, "rook-ceph-operator", 0)
+	err = k8sutil.SetDeploymentScale(ctx, k8sclientset.Kube, operatorNamespace, operatorName, 0)
 	if err != nil {
-		deploy, er := k8sutil.GetDeployment(ctx, k8sclientset.Kube, operatorNamespace, "rook-ceph-operator")
+		deploy, er := k8sutil.GetDeployment(ctx, k8sclientset.Kube, operatorNamespace, operatorName)
 		if er != nil && !apierrors.IsNotFound(er) {
 			logging.Fatal(er)
 		}
@@ -94,12 +101,12 @@ func RestoreCrd(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorN
 		logging.Fatal(fmt.Errorf("failed to delete validating webhook %s. %v", webhookConfigName, err))
 	}
 
-	removeOwnerRefOfUID(ctx, k8sclientset, operatorNamespace, clusterNamespace, string(crdResource.GetUID()))
+	removeOwnerRefOfUID(ctx, k8sclientset, operatorNamespace, clusterNamespace, string(crdResource.GetUID()), customResources)
 
 	logging.Info("Removing finalizers from %s/%s", crd, crName)
 
 	jsonPatchData, _ := json.Marshal(crds.DefaultResourceRemoveFinalizers)
-	err = k8sclientset.PatchResourcesDynamically(ctx, crds.CephRookIoGroup, crds.CephRookResourcesVersion, crd, clusterNamespace, crName, types.MergePatchType, jsonPatchData)
+	err = k8sclientset.PatchResourcesDynamically(ctx, groupName, versionResource, crd, clusterNamespace, crName, types.MergePatchType, jsonPatchData)
 	if err != nil {
 		logging.Fatal(fmt.Errorf("Failed to update resource %q for crd. %v", crName, err))
 	}
@@ -109,13 +116,13 @@ func RestoreCrd(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorN
 	crdResource.SetSelfLink("")
 	crdResource.SetCreationTimestamp(v1.Time{})
 	logging.Info("Re-creating the CR %s from dynamic resource", crd)
-	_, err = k8sclientset.CreateResourcesDynamically(ctx, crds.CephRookIoGroup, crds.CephRookResourcesVersion, crd, &crdResource, clusterNamespace)
+	_, err = k8sclientset.CreateResourcesDynamically(ctx, groupName, versionResource, crd, &crdResource, clusterNamespace)
 	if err != nil {
 		logging.Fatal((fmt.Errorf("Failed to create updated resource %q for crd. %v", crName, err)))
 	}
 
 	logging.Info("Scaling up the operator")
-	err = k8sutil.SetDeploymentScale(ctx, k8sclientset.Kube, operatorNamespace, "rook-ceph-operator", 1)
+	err = k8sutil.SetDeploymentScale(ctx, k8sclientset.Kube, operatorNamespace, operatorName, 1)
 	if err != nil {
 		logging.Fatal(errors.Wrapf(err, "Operator pod still being scaled up"))
 	}
@@ -123,9 +130,46 @@ func RestoreCrd(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorN
 	logging.Info("CR is successfully restored. Please watch the operator logs and check the crd")
 }
 
-func removeOwnerRefOfUID(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorNamespace, clusterNamespace, targetUID string) {
+func removeOwnerRefOfUID(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorNamespace, clusterNamespace, targetUID string, customResources []CustomResource) {
 	logging.Info("Removing ownerreferences from resources with matching uid %s", targetUID)
 
+	for _, crd := range customResources {
+		gvr := schema.GroupVersionResource{
+			Group:    crd.Group,
+			Version:  crd.Version,
+			Resource: crd.Resource,
+		}
+
+		crList, err := k8sclientset.ListResourcesDynamically(ctx, crd.Group, crd.Version, crd.Resource, clusterNamespace)
+		if err != nil {
+			logging.Warning("Failed to list %s: %v", crd.Resource, err)
+			continue
+		}
+
+		for _, cr := range crList {
+			updated := false
+			owners := cr.GetOwnerReferences()
+			newOwners := []v1.OwnerReference{}
+
+			for _, owner := range owners {
+				if string(owner.UID) != targetUID {
+					newOwners = append(newOwners, owner)
+				} else {
+					updated = true
+				}
+			}
+
+			if updated {
+				logging.Info("Removing ownerReference for %s/%s", crd.Resource, cr.GetName())
+				cr.SetOwnerReferences(newOwners)
+
+				_, err := k8sclientset.Dynamic.Resource(gvr).Namespace(clusterNamespace).Update(ctx, &cr, v1.UpdateOptions{})
+				if err != nil {
+					logging.Fatal(fmt.Errorf("Failed to update ownerReferences for %s/%s: %v", crd.Resource, cr.GetName(), err))
+				}
+			}
+		}
+	}
 	secrets, err := k8sclientset.Kube.CoreV1().Secrets(clusterNamespace).List(ctx, v1.ListOptions{})
 	if err != nil {
 		logging.Fatal(errors.Wrapf(err, "Failed to list secrets"))


Reviews:
user: login: parth-gr
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T16:03:44Z
user: login: travisn
body: To summarize some offline discussion in huddle... Since we're essentially supporting the kubectl plugin API, this change looks good. While we don't want downstream requirements creeping in, this really is just enhancing the upstream API even if the upstream tool doesn't use the feature. 

Just one whitespace nit...
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T18:33:45Z
user: login: OdedViner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T13:33:15Z
user: login: OdedViner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T13:44:35Z
user: login: travisn
body: @subhamkrai Another flaky test issue?
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-23T17:37:59Z
user: login: subhamkrai
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-28T05:02:21Z

Review comments:
diff_hunk: @@ -32,6 +32,7 @@ var RestoreCmd = &cobra.Command{
 		verifyOperatorPodIsRunning(cmd.Context(), clientSets)
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		restore.RestoreCrd(cmd.Context(), clientSets, operatorNamespace, cephClusterNamespace, crds.CephRookIoGroup, crds.CephRookResourcesVersion, args)
+		customResources := []restore.CustomResource{}
+		restore.RestoreCrd(cmd.Context(), clientSets, operatorNamespace, cephClusterNamespace, crds.CephRookIoGroup, crds.CephRookResourcesVersion, "rook-ceph-operator", customResources, args)
path: cmd/commands/restore.go
user: login: parth-gr
body: do we need to update the doc cmd with this?
created_at: 2025-04-22T16:03:44Z
author_association: MEMBER
diff_hunk: @@ -32,6 +32,7 @@ var RestoreCmd = &cobra.Command{
 		verifyOperatorPodIsRunning(cmd.Context(), clientSets)
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		restore.RestoreCrd(cmd.Context(), clientSets, operatorNamespace, cephClusterNamespace, crds.CephRookIoGroup, crds.CephRookResourcesVersion, args)
+		customResources := []restore.CustomResource{}
+		restore.RestoreCrd(cmd.Context(), clientSets, operatorNamespace, cephClusterNamespace, crds.CephRookIoGroup, crds.CephRookResourcesVersion, "rook-ceph-operator", customResources, args)
path: cmd/commands/restore.go
user: login: travisn
body: No change needed for users of the kubectl plugin, since the rook operator is still being stopped as before. It's only API consumers that have a change, but we don't have documentation for the API.
created_at: 2025-04-22T18:30:30Z
author_association: MEMBER
diff_hunk: @@ -94,38 +101,75 @@ func RestoreCrd(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorN
 		logging.Fatal(fmt.Errorf("failed to delete validating webhook %s. %v", webhookConfigName, err))
 	}
 
-	removeOwnerRefOfUID(ctx, k8sclientset, operatorNamespace, clusterNamespace, string(crdResource.GetUID()))
+	removeOwnerRefOfUID(ctx, k8sclientset, operatorNamespace, clusterNamespace, string(crdResource.GetUID()), customResources)
 
 	logging.Info("Removing finalizers from %s/%s", crd, crName)
 
 	jsonPatchData, _ := json.Marshal(crds.DefaultResourceRemoveFinalizers)
-	err = k8sclientset.PatchResourcesDynamically(ctx, crds.CephRookIoGroup, crds.CephRookResourcesVersion, crd, clusterNamespace, crName, types.MergePatchType, jsonPatchData)
+	err = k8sclientset.PatchResourcesDynamically(ctx, groupName, versionResource, crd, clusterNamespace, crName, types.MergePatchType, jsonPatchData)
 	if err != nil {
-		logging.Fatal(fmt.Errorf("Failed to update resource %q for crd. %v", crName, err))
+		logging.Fatal(fmt.Errorf("Failed to update resource %q for crd.  %v", crName, err))
path: pkg/restore/crd.go
user: login: travisn
body: nit: no need for extra space
```suggestion
		logging.Fatal(fmt.Errorf("Failed to update resource %q for crd. %v", crName, err))
```
created_at: 2025-04-22T18:31:10Z
author_association: MEMBER
diff_hunk: @@ -94,38 +101,75 @@ func RestoreCrd(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorN
 		logging.Fatal(fmt.Errorf("failed to delete validating webhook %s. %v", webhookConfigName, err))
 	}
 
-	removeOwnerRefOfUID(ctx, k8sclientset, operatorNamespace, clusterNamespace, string(crdResource.GetUID()))
+	removeOwnerRefOfUID(ctx, k8sclientset, operatorNamespace, clusterNamespace, string(crdResource.GetUID()), customResources)
 
 	logging.Info("Removing finalizers from %s/%s", crd, crName)
 
 	jsonPatchData, _ := json.Marshal(crds.DefaultResourceRemoveFinalizers)
-	err = k8sclientset.PatchResourcesDynamically(ctx, crds.CephRookIoGroup, crds.CephRookResourcesVersion, crd, clusterNamespace, crName, types.MergePatchType, jsonPatchData)
+	err = k8sclientset.PatchResourcesDynamically(ctx, groupName, versionResource, crd, clusterNamespace, crName, types.MergePatchType, jsonPatchData)
 	if err != nil {
-		logging.Fatal(fmt.Errorf("Failed to update resource %q for crd. %v", crName, err))
+		logging.Fatal(fmt.Errorf("Failed to update resource %q for crd.  %v", crName, err))
path: pkg/restore/crd.go
user: login: OdedViner
body: done.
created_at: 2025-04-23T13:33:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -94,38 +101,75 @@ func RestoreCrd(ctx context.Context, k8sclientset *k8sutil.Clientsets, operatorN
 		logging.Fatal(fmt.Errorf("failed to delete validating webhook %s. %v", webhookConfigName, err))
 	}
 
-	removeOwnerRefOfUID(ctx, k8sclientset, operatorNamespace, clusterNamespace, string(crdResource.GetUID()))
+	removeOwnerRefOfUID(ctx, k8sclientset, operatorNamespace, clusterNamespace, string(crdResource.GetUID()), customResources)
 
 	logging.Info("Removing finalizers from %s/%s", crd, crName)
 
 	jsonPatchData, _ := json.Marshal(crds.DefaultResourceRemoveFinalizers)
-	err = k8sclientset.PatchResourcesDynamically(ctx, crds.CephRookIoGroup, crds.CephRookResourcesVersion, crd, clusterNamespace, crName, types.MergePatchType, jsonPatchData)
+	err = k8sclientset.PatchResourcesDynamically(ctx, groupName, versionResource, crd, clusterNamespace, crName, types.MergePatchType, jsonPatchData)
 	if err != nil {
-		logging.Fatal(fmt.Errorf("Failed to update resource %q for crd. %v", crName, err))
+		logging.Fatal(fmt.Errorf("Failed to update resource %q for crd.  %v", crName, err))
path: pkg/restore/crd.go
user: login: OdedViner
body: I added this space only to verify that the changes are reflected when importing into the odf-cli repo [forked] from my private branch of the kubectl-rook-ceph repo[forked]. I tested the function using odf-cli with go mod vendor—this was solely for testing purposes.
https://github.com/red-hat-storage/odf-cli/pull/114/files#diff-33ef32bf6c23acb95f5902d7097b7a1d5128ca061167ec0716715b0b9eeaa5f6R135
created_at: 2025-04-23T13:44:35Z
author_association: CONTRIBUTOR

PR comments:
user: login: subhamkrai
created_at: 2025-04-22T05:51:18Z
body: @OdedViner please rebase your PR to get CI fixes. Thanks
author_association: COLLABORATOR
user: login: subhamkrai
created_at: 2025-04-28T05:02:43Z
body: > @subhamkrai Another flaky test issue?

Yes, restarting CI worked.
author_association: COLLABORATOR