Repository: rook/rook (2025-07-02T18:45:02Z)

Issue:
Disruption controller does not gracefully handle missing k8s nodes
<!-- **Are you in the right place?**
1. For issues or feature requests, please create an issue in this repository.
2. For general technical and non-technical questions, we are happy to help you on our [Rook.io Slack](https://slack.rook.io/).
3. Did you already search the existing open issues for anything similar? -->

**Is this a bug report or feature request?**
* Bug Report

**Deviation from expected behavior:**
When the disruption controller is determining failure domains for OSDs it does not gracefully handle when a node does not exist within kubernetes.

When checking if an OSD node has drained it is possible that the node longer exists and the call to k8s returns a `NOTFOUND` error code. In that case the error is propagated up and we propagate the error to the caller of getOSDFailureDomains. This means that if a node is missing from the k8s cluster it will indefinitely block any OSD PDB updates:
https://github.com/rook/rook/blob/029c3453727952754f7bf4e2abbb535443f05f1f/pkg/operator/ceph/disruption/clusterdisruption/osd.go#L462

This can cause data availability issues like so:
1. An OSD node is mid drain, so we have set per host PDBs as we are mid drain
2. A k8s node goes missing and no longer exists
3. All further OSD PDB reconciliations fail
4. We add some new OSDs to the cluster
5. Those new OSDs do not get per host PDBs as PDB reconciliation is failing
6. Something causes a cluster disruption which disrupts multiple of the new OSDs which are not protected by PDBs
7. If all replicas for a PDB exist on the new unprotected OSDs, the pg can go down

**Expected behavior:**
We should treat k8s nodes which no longer exist as drained.

**Environment**:
* Rook version (use `rook version` inside of a Rook Pod): `v1.17`



Discussion:
@elias-dbx Can you share the rook operator log that shows this case with the pdb reconcile is failing?
@travisn 
```
clusterdisruption-controller: failed to check if osd "rook-ceph-osd-133" node is drained: failed to get node "pdx4a-rj9-7a": failed to get node "pdx4a-rj9-7a": Node "pdx4a-rj9-7a" not found
```



Patch:
diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd.go b/pkg/operator/ceph/disruption/clusterdisruption/osd.go
index 5a4979e47333..42e46d18e19f 100644
--- a/pkg/operator/ceph/disruption/clusterdisruption/osd.go
+++ b/pkg/operator/ceph/disruption/clusterdisruption/osd.go
@@ -490,6 +490,9 @@ func (r *ReconcileClusterDisruption) getOSDFailureDomains(clusterInfo *cephclien
 // hasOSDNodeDrained returns true if OSD pod is not assigned to any node or if the OSD node is not schedulable
 func hasOSDNodeDrained(ctx context.Context, c client.Client, osdNodeName string) (bool, error) {
 	node, err := getNode(ctx, c, osdNodeName)
+	if apierrors.IsNotFound(err) {
+		return true, nil
+	}
 	if err != nil {
 		return false, errors.Wrapf(err, "failed to get node %q", osdNodeName)
 	}


Test patch:
diff --git a/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go b/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go
index 754e6edfd57a..e1ffa441fd34 100644
--- a/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go
+++ b/pkg/operator/ceph/disruption/clusterdisruption/osd_test.go
@@ -181,6 +181,30 @@ func TestGetOSDFailureDomains(t *testing.T) {
 			expectedDrainingFailureDomains: []string{"zone-3"},
 			expectedDownOSDs:               []int{3},
 		},
+		{
+			name: "case 6: osd in zone-3 is pending and the osd node does not exist",
+			osds: []appsv1.Deployment{
+				fakeOSDDeployment(1, 1), fakeOSDDeployment(2, 1),
+				fakeOSDDeployment(3, 0),
+			},
+			nodes:                          []corev1.Node{*getNodeObject("node-1", false), *getNodeObject("node-2", false)},
+			expectedAllFailureDomains:      []string{"zone-1", "zone-2", "zone-3"},
+			expectedOsdDownFailureDomains:  []string{"zone-3"},
+			expectedDrainingFailureDomains: []string{"zone-3"},
+			expectedDownOSDs:               []int{3},
+		},
+		{
+			name: "case 7: osd in zone-1 and zone-2 are pending and node does not exist",
+			osds: []appsv1.Deployment{
+				fakeOSDDeployment(1, 0), fakeOSDDeployment(2, 0),
+				fakeOSDDeployment(3, 1),
+			},
+			nodes:                          []corev1.Node{*getNodeObject("node-3", false)},
+			expectedAllFailureDomains:      []string{"zone-1", "zone-2", "zone-3"},
+			expectedOsdDownFailureDomains:  []string{"zone-1", "zone-2"},
+			expectedDrainingFailureDomains: []string{"zone-1", "zone-2"},
+			expectedDownOSDs:               []int{1, 2},
+		},
 	}
 
 	for _, tc := range testcases {
@@ -426,11 +450,18 @@ func TestHasNodeDrained(t *testing.T) {
 	assert.False(t, expected)
 
 	// Expecting node drain because OSD pod is assigned to an unschedulable node
-	osdDeployment = fakeOSDDeployment(2, 2)
+	osdDeployment = fakeOSDDeployment(2, 0)
 	r = getFakeReconciler(t, getNodeObject("node-2", true), osdDeployment.DeepCopy(), &corev1.ConfigMap{})
 	expected, err = hasOSDNodeDrained(ctx, r.client, "node-2")
 	assert.NoError(t, err)
 	assert.True(t, expected)
+
+	// Expecting node drain because OSD pod is assigned to a non existent node
+	osdDeployment = fakeOSDDeployment(3, 0)
+	r = getFakeReconciler(t, osdDeployment.DeepCopy(), &corev1.ConfigMap{})
+	expected, err = hasOSDNodeDrained(ctx, r.client, "node-3")
+	assert.NoError(t, err)
+	assert.True(t, expected)
 }
 
 func TestSetPDBConfig(t *testing.T) {