Repository: NVIDIA/gpu-operator (2026-03-31T14:34:02Z)

Issue:
[Bug]: NVIDIADriver CRs remain stuck after deleting conflicting default CR due to missing requeue of peer NVIDIADriver resources
_**Important Note:  NVIDIA AI Enterprise customers can get support from NVIDIA Enterprise support. Please open a case [here](https://enterprise-support.nvidia.com/s/create-case)**._

**Describe the bug**
When multiple NVIDIADriver CRs are present, conflict validation is evaluated globally across all CR node selectors. If the default NVIDIADriver (no explicit selector) overlaps with custom CRs, custom CRs fail validation and exit reconcile without requeue.
After deleting the default/conflicting CR, the previously blocked CRs do not automatically reconcile, so they remain stuck until a separate unrelated event triggers them.

**To Reproduce**
Deploy GPU Operator with default NVIDIADriver CR.
Create 2+ custom NVIDIADriver CRs with unique [nodeSelector](vscode-file://vscode-app/Applications/Visual%20Studio%20Code.app/Contents/Resources/app/out/vs/code/electron-browser/workbench/workbench.html) labels.
Label nodes to match custom CR selectors.
Observe custom CRs are blocked due to conflict with default CR.
Delete the default NVIDIADriver CR.
Observe blocked custom CRs do not progress automatically.

**Expected behavior**
Deleting/updating a NVIDIADriver CR that changes global conflict state should trigger reconciliation of other NVIDIADriver CRs so they revalidate and proceed.

**Environment (please provide the following information):**
 - GPU Operator Version: v26.3.0
 - OS: ubuntu24.04, ubuntu22.04
 - Kubernetes Distro and Version: kubeadm 1.34

Collecting full debug bundle (optional):

```
curl -o must-gather.sh -L https://raw.githubusercontent.com/NVIDIA/gpu-operator/main/hack/must-gather.sh
chmod +x must-gather.sh
./must-gather.sh
```
**NOTE**: please refer to the [must-gather](https://raw.githubusercontent.com/NVIDIA/gpu-operator/main/hack/must-gather.sh) script for debug data collected.

This bundle can be submitted to us via email: **operator_feedback@nvidia.com**



Discussion:
Given we don't have any customers reporting this, we are going to have the fix in v26.7.0



Patch:
diff --git a/controllers/nvidiadriver_controller.go b/controllers/nvidiadriver_controller.go
index a53a8262e..637b9cec9 100644
--- a/controllers/nvidiadriver_controller.go
+++ b/controllers/nvidiadriver_controller.go
@@ -249,6 +249,33 @@ func (r *NVIDIADriverReconciler) updateCrStatus(
 	return nil
 }
 
+// enqueueAllNVIDIADrivers lists all NVIDIADriver instances in the cluster and enqueues a reconcile
+// request for each instance. This is used to trigger reconciliation for all NVIDIADriver instances
+// when a relevant event occurs (e.g. ClusterPolicy/NVIDIADriver update, node label change, etc).
+func (r *NVIDIADriverReconciler) enqueueAllNVIDIADrivers(ctx context.Context) []reconcile.Request {
+	logger := log.FromContext(ctx)
+	list := &nvidiav1alpha1.NVIDIADriverList{}
+
+	err := r.List(ctx, list)
+	if err != nil {
+		logger.Error(err, "Unable to list NVIDIADriver resources")
+		return []reconcile.Request{}
+	}
+
+	reconcileRequests := make([]reconcile.Request, 0, len(list.Items))
+	for _, nvidiaDriver := range list.Items {
+		reconcileRequests = append(reconcileRequests,
+			reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Name:      nvidiaDriver.GetName(),
+					Namespace: nvidiaDriver.GetNamespace(),
+				},
+			})
+	}
+
+	return reconcileRequests
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	// Create state manager
@@ -278,11 +305,17 @@ func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.
 		return err
 	}
 
-	// Watch for changes to the primary resource NVIDIaDriver
+	// Watch for changes to NVIDIADriver CRs. Whenever an event is generated for a NVIDIADriver CR,
+	// enqueue a reconcile request for all NVIDIADriver instances.
+	nvidiaDriverMapFn := func(ctx context.Context, _ *nvidiav1alpha1.NVIDIADriver) []reconcile.Request {
+		return r.enqueueAllNVIDIADrivers(ctx)
+	}
+
+	// Watch for changes to the primary resource NVIDIADriver
 	err = c.Watch(source.Kind(
 		mgr.GetCache(),
 		&nvidiav1alpha1.NVIDIADriver{},
-		&handler.TypedEnqueueRequestForObject[*nvidiav1alpha1.NVIDIADriver]{},
+		handler.TypedEnqueueRequestsFromMapFunc(nvidiaDriverMapFn),
 		predicate.TypedGenerationChangedPredicate[*nvidiav1alpha1.NVIDIADriver]{},
 	),
 	)
@@ -292,63 +325,21 @@ func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.
 
 	// Watch for changes to ClusterPolicy. Whenever an event is generated for ClusterPolicy, enqueue
 	// a reconcile request for all NVIDIADriver instances.
-	mapFn := func(ctx context.Context, cp *gpuv1.ClusterPolicy) []reconcile.Request {
-		logger := log.FromContext(ctx)
-		opts := []client.ListOption{}
-		list := &nvidiav1alpha1.NVIDIADriverList{}
-
-		err := mgr.GetClient().List(ctx, list, opts...)
-		if err != nil {
-			logger.Error(err, "Unable to list NVIDIADriver resources")
-			return []reconcile.Request{}
-		}
-
-		reconcileRequests := []reconcile.Request{}
-		for _, nvidiaDriver := range list.Items {
-			reconcileRequests = append(reconcileRequests,
-				reconcile.Request{
-					NamespacedName: types.NamespacedName{
-						Name:      nvidiaDriver.GetName(),
-						Namespace: nvidiaDriver.GetNamespace(),
-					},
-				})
-		}
-
-		return reconcileRequests
+	mapFn := func(ctx context.Context, _ *gpuv1.ClusterPolicy) []reconcile.Request {
+		return r.enqueueAllNVIDIADrivers(ctx)
 	}
 
-	// Watch for changes to the Nodes. Whenever an event is generated for ClusterPolicy, enqueue
+	// Watch for changes to the Nodes. Whenever an event is generated for a Node, enqueue
 	// a reconcile request for all NVIDIADriver instances.
-	nodeMapFn := func(ctx context.Context, cp *corev1.Node) []reconcile.Request {
-		logger := log.FromContext(ctx)
-		opts := []client.ListOption{}
-		list := &nvidiav1alpha1.NVIDIADriverList{}
-
-		err := mgr.GetClient().List(ctx, list, opts...)
-		if err != nil {
-			logger.Error(err, "Unable to list NVIDIADriver resources")
-			return []reconcile.Request{}
-		}
-
-		reconcileRequests := []reconcile.Request{}
-		for _, nvidiaDriver := range list.Items {
-			reconcileRequests = append(reconcileRequests,
-				reconcile.Request{
-					NamespacedName: types.NamespacedName{
-						Name:      nvidiaDriver.GetName(),
-						Namespace: nvidiaDriver.GetNamespace(),
-					},
-				})
-		}
-
-		return reconcileRequests
+	nodeMapFn := func(ctx context.Context, _ *corev1.Node) []reconcile.Request {
+		return r.enqueueAllNVIDIADrivers(ctx)
 	}
 
 	err = c.Watch(
 		source.Kind(
 			mgr.GetCache(),
 			&gpuv1.ClusterPolicy{},
-			handler.TypedEnqueueRequestsFromMapFunc[*gpuv1.ClusterPolicy](mapFn),
+			handler.TypedEnqueueRequestsFromMapFunc(mapFn),
 			predicate.TypedGenerationChangedPredicate[*gpuv1.ClusterPolicy]{},
 		),
 	)
@@ -386,7 +377,7 @@ func (r *NVIDIADriverReconciler) SetupWithManager(ctx context.Context, mgr ctrl.
 	err = c.Watch(
 		source.Kind(mgr.GetCache(),
 			&corev1.Node{},
-			handler.TypedEnqueueRequestsFromMapFunc[*corev1.Node](nodeMapFn),
+			handler.TypedEnqueueRequestsFromMapFunc(nodeMapFn),
 			nodePredicate,
 		),
 	)


Test patch:
diff --git a/controllers/nvidiadriver_controller_test.go b/controllers/nvidiadriver_controller_test.go
index 243812d32..7630e7e4b 100644
--- a/controllers/nvidiadriver_controller_test.go
+++ b/controllers/nvidiadriver_controller_test.go
@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"sort"
 	"testing"
 
 	"github.com/go-logr/logr"
@@ -245,3 +246,24 @@ func TestReconcileConflictSetsNotReadyState(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, nvidiav1alpha1.NotReady, updater.LastErrorState)
 }
+
+func TestEnqueueAllNVIDIADrivers(t *testing.T) {
+	scheme := runtime.NewScheme()
+	require.NoError(t, nvidiav1alpha1.AddToScheme(scheme))
+
+	client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+		&nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-a", Namespace: "default"}},
+		&nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-b", Namespace: "default"}},
+	).Build()
+
+	reconciler := &NVIDIADriverReconciler{Client: client}
+	requests := reconciler.enqueueAllNVIDIADrivers(context.Background())
+
+	require.Len(t, requests, 2)
+	got := []string{
+		requests[0].String(),
+		requests[1].String(),
+	}
+	sort.Strings(got)
+	require.Equal(t, []string{"default/driver-a", "default/driver-b"}, got)
+}