Repository: kubernetes/kops (2025-07-27T07:40:10Z)

Issue:
Support passing image tag version to nvidia k8s-device-plugin
/kind feature

**1. Describe IN DETAIL the feature/behavior/change you would like to see.**
Similar issue described earlier https://github.com/kubernetes/kops/issues/14103
**2. Feel free to provide a design supporting your feature request.**


Would be beneficial if we can pass the tag. Its behind some versions now. One important thing I faced is I couldn't override the config properly (not sure, this might be there or people can help me more)

The config was the append timeSlicing feature for the gpus replicas. (https://github.com/NVIDIA/k8s-device-plugin?tab=readme-ov-file#shared-access-to-gpus)

Right now, manually patching the DS worked. But running kops update may override that again. 

Any plan to support this? Right now they have 0.17.2 And kops maybe hardcoded v0.12.2 



Discussion:
Hi, I'm interested in working on this issue. Could someone please confirm if it's been triaged and accepted?
Yes, feel free to open a PR
/assign



Patch:
diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
index 1aff75953996f..99763ee047c7a 100644
--- a/k8s/crds/kops.k8s.io_clusters.yaml
+++ b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -932,10 +932,14 @@ spec:
                           Enabled determines if kOps will install the Nvidia GPU runtime and drivers.
                           They will only be installed on intances that has an Nvidia GPU.
                         type: boolean
+                      image:
+                        description: Image defines the container image used to deploy
+                          the Nvidia Kubernetes Device Plugin.
+                        type: string
                       package:
                         description: |-
                           Package is the name of the nvidia driver package that will be installed.
-                          Default is "nvidia-headless-460-server".
+                          Default is "nvidia-driver-535-server".
                         type: string
                     type: object
                   packages:
diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml
index c90e8426a4cdb..3d18bdfb26552 100644
--- a/k8s/crds/kops.k8s.io_instancegroups.yaml
+++ b/k8s/crds/kops.k8s.io_instancegroups.yaml
@@ -168,10 +168,14 @@ spec:
                           Enabled determines if kOps will install the Nvidia GPU runtime and drivers.
                           They will only be installed on intances that has an Nvidia GPU.
                         type: boolean
+                      image:
+                        description: Image defines the container image used to deploy
+                          the Nvidia Kubernetes Device Plugin.
+                        type: string
                       package:
                         description: |-
                           Package is the name of the nvidia driver package that will be installed.
-                          Default is "nvidia-headless-460-server".
+                          Default is "nvidia-driver-535-server".
                         type: string
                     type: object
                   packages:
diff --git a/pkg/apis/kops/containerdconfig.go b/pkg/apis/kops/containerdconfig.go
index 9efe76151798c..4e3e182e9b431 100644
--- a/pkg/apis/kops/containerdconfig.go
+++ b/pkg/apis/kops/containerdconfig.go
@@ -21,8 +21,12 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
-// NvidiaDefaultDriverPackage is the nvidia driver default version
-const NvidiaDefaultDriverPackage = "nvidia-driver-535-server"
+const (
+	// NvidiaDefaultDriverPackage is the nvidia driver default version
+	NvidiaDefaultDriverPackage = "nvidia-driver-535-server"
+	// NvidiaDevicePluginImage is the Nvidia K8s device plugin container image
+	NvidiaDevicePluginImage = "nvcr.io/nvidia/k8s-device-plugin:v0.17.3"
+)
 
 // ContainerdConfig is the configuration for containerd
 type ContainerdConfig struct {
@@ -67,11 +71,13 @@ type NRIConfig struct {
 
 type NvidiaGPUConfig struct {
 	// Package is the name of the nvidia driver package that will be installed.
-	// Default is "nvidia-headless-510-server".
+	// Default is "nvidia-driver-535-server".
 	DriverPackage string `json:"package,omitempty"`
 	// Enabled determines if kOps will install the Nvidia GPU runtime and drivers.
-	// They will only be installed on intances that has an Nvidia GPU.
+	// They will only be installed on instances that has an Nvidia GPU.
 	Enabled *bool `json:"enabled,omitempty"`
+	// Image defines the container image used to deploy the Nvidia Kubernetes Device Plugin.
+	DevicePluginImage string `json:"image,omitempty"`
 	// DCGMExporterConfig configures the DCGM exporter
 	DCGMExporter *DCGMExporterConfig `json:"dcgmExporter,omitempty"`
 }
diff --git a/pkg/apis/kops/v1alpha2/containerdconfig.go b/pkg/apis/kops/v1alpha2/containerdconfig.go
index e4430b7908bf8..af034b9872eba 100644
--- a/pkg/apis/kops/v1alpha2/containerdconfig.go
+++ b/pkg/apis/kops/v1alpha2/containerdconfig.go
@@ -64,11 +64,13 @@ type NRIConfig struct {
 
 type NvidiaGPUConfig struct {
 	// Package is the name of the nvidia driver package that will be installed.
-	// Default is "nvidia-headless-460-server".
+	// Default is "nvidia-driver-535-server".
 	DriverPackage string `json:"package,omitempty"`
 	// Enabled determines if kOps will install the Nvidia GPU runtime and drivers.
 	// They will only be installed on intances that has an Nvidia GPU.
 	Enabled *bool `json:"enabled,omitempty"`
+	// Image defines the container image used to deploy the Nvidia Kubernetes Device Plugin.
+	DevicePluginImage string `json:"image,omitempty"`
 	// DCGMExporterConfig configures the DCGM exporter
 	DCGMExporter *DCGMExporterConfig `json:"dcgmExporter,omitempty"`
 }
diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
index 0365638fe7b18..cfb7b744d9e5c 100644
--- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
@@ -6775,6 +6775,7 @@ func Convert_kops_NodeTerminationHandlerSpec_To_v1alpha2_NodeTerminationHandlerS
 func autoConvert_v1alpha2_NvidiaGPUConfig_To_kops_NvidiaGPUConfig(in *NvidiaGPUConfig, out *kops.NvidiaGPUConfig, s conversion.Scope) error {
 	out.DriverPackage = in.DriverPackage
 	out.Enabled = in.Enabled
+	out.DevicePluginImage = in.DevicePluginImage
 	if in.DCGMExporter != nil {
 		in, out := &in.DCGMExporter, &out.DCGMExporter
 		*out = new(kops.DCGMExporterConfig)
@@ -6795,6 +6796,7 @@ func Convert_v1alpha2_NvidiaGPUConfig_To_kops_NvidiaGPUConfig(in *NvidiaGPUConfi
 func autoConvert_kops_NvidiaGPUConfig_To_v1alpha2_NvidiaGPUConfig(in *kops.NvidiaGPUConfig, out *NvidiaGPUConfig, s conversion.Scope) error {
 	out.DriverPackage = in.DriverPackage
 	out.Enabled = in.Enabled
+	out.DevicePluginImage = in.DevicePluginImage
 	if in.DCGMExporter != nil {
 		in, out := &in.DCGMExporter, &out.DCGMExporter
 		*out = new(DCGMExporterConfig)
diff --git a/pkg/apis/kops/v1alpha3/containerdconfig.go b/pkg/apis/kops/v1alpha3/containerdconfig.go
index be69f3e9a7e51..1fdc5692163ac 100644
--- a/pkg/apis/kops/v1alpha3/containerdconfig.go
+++ b/pkg/apis/kops/v1alpha3/containerdconfig.go
@@ -64,11 +64,13 @@ type NRIConfig struct {
 
 type NvidiaGPUConfig struct {
 	// Package is the name of the nvidia driver package that will be installed.
-	// Default is "nvidia-headless-460-server".
+	// Default is "nvidia-driver-535-server".
 	DriverPackage string `json:"package,omitempty"`
 	// Enabled determines if kOps will install the Nvidia GPU runtime and drivers.
 	// They will only be installed on intances that has an Nvidia GPU.
 	Enabled *bool `json:"enabled,omitempty"`
+	// Image defines the container image used to deploy the Nvidia Kubernetes Device Plugin.
+	DevicePluginImage string `json:"image,omitempty"`
 	// DCGMExporterConfig configures the DCGM exporter
 	DCGMExporter *DCGMExporterConfig `json:"dcgmExporter,omitempty"`
 }
diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
index cbf6974d27981..9bd4842628a1b 100644
--- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
@@ -7032,6 +7032,7 @@ func Convert_kops_NodeTerminationHandlerSpec_To_v1alpha3_NodeTerminationHandlerS
 func autoConvert_v1alpha3_NvidiaGPUConfig_To_kops_NvidiaGPUConfig(in *NvidiaGPUConfig, out *kops.NvidiaGPUConfig, s conversion.Scope) error {
 	out.DriverPackage = in.DriverPackage
 	out.Enabled = in.Enabled
+	out.DevicePluginImage = in.DevicePluginImage
 	if in.DCGMExporter != nil {
 		in, out := &in.DCGMExporter, &out.DCGMExporter
 		*out = new(kops.DCGMExporterConfig)
@@ -7052,6 +7053,7 @@ func Convert_v1alpha3_NvidiaGPUConfig_To_kops_NvidiaGPUConfig(in *NvidiaGPUConfi
 func autoConvert_kops_NvidiaGPUConfig_To_v1alpha3_NvidiaGPUConfig(in *kops.NvidiaGPUConfig, out *NvidiaGPUConfig, s conversion.Scope) error {
 	out.DriverPackage = in.DriverPackage
 	out.Enabled = in.Enabled
+	out.DevicePluginImage = in.DevicePluginImage
 	if in.DCGMExporter != nil {
 		in, out := &in.DCGMExporter, &out.DCGMExporter
 		*out = new(DCGMExporterConfig)
diff --git a/pkg/model/components/containerd.go b/pkg/model/components/containerd.go
index eda4e46124919..456670385f8e5 100644
--- a/pkg/model/components/containerd.go
+++ b/pkg/model/components/containerd.go
@@ -57,8 +57,14 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o *kops.Cluster) error {
 	// Set default log level to INFO
 	containerd.LogLevel = fi.PtrTo("info")
 
-	if containerd.NvidiaGPU != nil && fi.ValueOf(containerd.NvidiaGPU.Enabled) && containerd.NvidiaGPU.DriverPackage == "" {
-		containerd.NvidiaGPU.DriverPackage = kops.NvidiaDefaultDriverPackage
+	if containerd.NvidiaGPU != nil && fi.ValueOf(containerd.NvidiaGPU.Enabled) {
+		if containerd.NvidiaGPU.DriverPackage == "" {
+			containerd.NvidiaGPU.DriverPackage = kops.NvidiaDefaultDriverPackage
+		}
+
+		if containerd.NvidiaGPU.DevicePluginImage == "" {
+			containerd.NvidiaGPU.DevicePluginImage = kops.NvidiaDevicePluginImage
+		}
 	}
 
 	return nil
diff --git a/upup/models/cloudup/resources/addons/nvidia.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/nvidia.addons.k8s.io/k8s-1.16.yaml.template
index 1aff485afbd34..c73aa1a34267b 100644
--- a/upup/models/cloudup/resources/addons/nvidia.addons.k8s.io/k8s-1.16.yaml.template
+++ b/upup/models/cloudup/resources/addons/nvidia.addons.k8s.io/k8s-1.16.yaml.template
@@ -17,7 +17,7 @@ spec:
         name: nvidia-device-plugin-ds
     spec:
       containers:
-      - image: nvcr.io/nvidia/k8s-device-plugin:v0.12.2
+      - image: {{ $.Containerd.NvidiaGPU.DevicePluginImage }}
         name: nvidia-device-plugin-ctr
         args: ["--fail-on-init-error=false"]
         securityContext:


Test patch:
diff --git a/tests/integration/update_cluster/nvidia/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/nvidia/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
index 50a552b9ff314..0205dfa14e01e 100644
--- a/tests/integration/update_cluster/nvidia/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
+++ b/tests/integration/update_cluster/nvidia/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data
@@ -130,7 +130,7 @@ ClusterName: minimal.example.com
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: ControlPlane
-NodeupConfigHash: erzIlDR9TAcOlxUIr7WouTREk7YdM0Ma4N2K9r5gk8w=
+NodeupConfigHash: e8ACnEK0qWqIMgJoxl9ojRfcJMBGi+zNVdLBnTkE0Go=
 
 __EOF_KUBE_ENV
 
diff --git a/tests/integration/update_cluster/nvidia/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/nvidia/data/aws_launch_template_nodes.minimal.example.com_user_data
index fb6146e67bf9e..40e51d9580e9f 100644
--- a/tests/integration/update_cluster/nvidia/data/aws_launch_template_nodes.minimal.example.com_user_data
+++ b/tests/integration/update_cluster/nvidia/data/aws_launch_template_nodes.minimal.example.com_user_data
@@ -153,7 +153,7 @@ ConfigServer:
   - https://kops-controller.internal.minimal.example.com:3988/
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: y10GCJ5TES8qY2D0+M5IOOVd3YWvQdbJk1LQxxA37pY=
+NodeupConfigHash: de+zCCygHUW6ieE/GHLXxzLD92pxl20/vODdPL2IbrA=
 
 __EOF_KUBE_ENV
 
diff --git a/tests/integration/update_cluster/nvidia/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/nvidia/data/aws_s3_object_cluster-completed.spec_content
index 17676a08e7550..d0f08ca3e794b 100644
--- a/tests/integration/update_cluster/nvidia/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/nvidia/data/aws_s3_object_cluster-completed.spec_content
@@ -28,6 +28,7 @@ spec:
     logLevel: info
     nvidiaGPU:
       enabled: true
+      image: nvcr.io/nvidia/k8s-device-plugin:v0.17.3
       package: nvidia-driver-535-server
     runc:
       version: 1.2.4
diff --git a/tests/integration/update_cluster/nvidia/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/nvidia/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
index 2353cae26c842..992e0c9dd3ce5 100644
--- a/tests/integration/update_cluster/nvidia/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/nvidia/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -92,7 +92,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: nvidia.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 5c2ea4e24c2272166a0bef02986a881160d23c39df8c7a5951a894702199d8a4
+    manifestHash: 6233f19a8b22836db6e6a25a121990ab77ea14c4e235a83004710afb515c8ff7
     name: nvidia.addons.k8s.io
     selector:
       k8s-addon: nvidia.addons.k8s.io
diff --git a/tests/integration/update_cluster/nvidia/data/aws_s3_object_minimal.example.com-addons-nvidia.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/nvidia/data/aws_s3_object_minimal.example.com-addons-nvidia.addons.k8s.io-k8s-1.16_content
index 8ec81ec634b90..821b3fdf7c9bd 100644
--- a/tests/integration/update_cluster/nvidia/data/aws_s3_object_minimal.example.com-addons-nvidia.addons.k8s.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/nvidia/data/aws_s3_object_minimal.example.com-addons-nvidia.addons.k8s.io-k8s-1.16_content
@@ -22,7 +22,7 @@ spec:
       containers:
       - args:
         - --fail-on-init-error=false
-        image: nvcr.io/nvidia/k8s-device-plugin:v0.12.2
+        image: nvcr.io/nvidia/k8s-device-plugin:v0.17.3
         name: nvidia-device-plugin-ctr
         securityContext:
           allowPrivilegeEscalation: false
diff --git a/tests/integration/update_cluster/nvidia/data/aws_s3_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/nvidia/data/aws_s3_object_nodeupconfig-master-us-test-1a_content
index 766b300c13500..fab3990d57279 100644
--- a/tests/integration/update_cluster/nvidia/data/aws_s3_object_nodeupconfig-master-us-test-1a_content
+++ b/tests/integration/update_cluster/nvidia/data/aws_s3_object_nodeupconfig-master-us-test-1a_content
@@ -308,6 +308,7 @@ Networking:
   serviceClusterIPRange: 100.64.0.0/13
 NvidiaGPU:
   enabled: true
+  image: nvcr.io/nvidia/k8s-device-plugin:v0.17.3
   package: nvidia-driver-535-server
 UpdatePolicy: automatic
 channels:
@@ -319,6 +320,7 @@ containerdConfig:
   logLevel: info
   nvidiaGPU:
     enabled: true
+    image: nvcr.io/nvidia/k8s-device-plugin:v0.17.3
     package: nvidia-driver-535-server
   runc:
     version: 1.2.4
diff --git a/tests/integration/update_cluster/nvidia/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/nvidia/data/aws_s3_object_nodeupconfig-nodes_content
index c407b5d051054..96bb10cb2d8ac 100644
--- a/tests/integration/update_cluster/nvidia/data/aws_s3_object_nodeupconfig-nodes_content
+++ b/tests/integration/update_cluster/nvidia/data/aws_s3_object_nodeupconfig-nodes_content
@@ -58,12 +58,14 @@ Networking:
   serviceClusterIPRange: 100.64.0.0/13
 NvidiaGPU:
   enabled: true
+  image: nvcr.io/nvidia/k8s-device-plugin:v0.17.3
   package: nvidia-driver-535-server
 UpdatePolicy: automatic
 containerdConfig:
   logLevel: info
   nvidiaGPU:
     enabled: true
+    image: nvcr.io/nvidia/k8s-device-plugin:v0.17.3
     package: nvidia-driver-535-server
   runc:
     version: 1.2.4