Repository: kubernetes/minikube
Pull request #15927 (2023-02-26 08:08:24)

container-runtime: add nvidia-docker
<!-- 🎉 Thank you for contributing to minikube! 🎉 Here are some hints to get your PR merged faster:

1. Your PR title will be included in the release notes, choose it carefully
2. If the PR fixes an issue, add "fixes #<issue number>" to the description.
3. If the PR is a user interface change, please include a "before" and "after" example.
4. If the PR is a large design change, please include an enhancement proposal:
https://github.com/kubernetes/minikube/tree/master/enhancements
-->

This adds a new container-runtime that sets the correct configuration options for using with https://github.com/NVIDIA/k8s-device-plugin#nvidia-device-plugin-for-kubernetes

This requires a custom Dockerfile with `nvidia-container-toolkit` and a matching `libnvidia-ml.so.1` file. The driver version on the host needs to exactly match the version of nvml in the container.

kicbase Dockerfile

```
FROM gcr.io/k8s-minikube/kicbase:v0.0.37

RUN curl -s -L https://nvidia.github.io/libnvidia-container/gpgkey | sudo apt-key add -
RUN curl -s -L https://nvidia.github.io/libnvidia-container/$(. /etc/os-release;echo $ID$VERSION_ID)/libnvidia-container.list | tee /etc/apt/sources.list.d/libnvidia-container.list
RUN apt-get update && sudo apt-get install -y nvidia-container-toolkit && rm -rf /var/lib/apt/lists/*

ADD libnvidia-ml.so.1 /usr/lib/libnvidia-ml.so.1
```

Commands to run:

```
# Build docker image
cp /usr/lib/libnvidia-ml.so.1 .
docker build -t nvidiakic .

# Example start minikube with nvidia-docker
go run ./cmd/minikube start --container-runtime nvidia-docker --base-image='nvidiakic' --iso-url='https://storage.googleapis.com/minikube/iso/minikube-v1.29.0-amd64.iso,https://github.com/kubernetes/minikube/releases/download/v1.29.0/minikube-v1.29.0-amd64.iso' --driver docker --cpus=max --memory=max --nodes=2

# install the normal k8s-deviced-plugin (not the minikube addon!)
kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.13.0/nvidia-device-plugin.yml
```

Related issue #10229


Patch:
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index 6edd86945..85488a7ee 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -50,7 +50,7 @@ func (cs ContainerState) String() string {
 
 // ValidRuntimes lists the supported container runtimes
 func ValidRuntimes() []string {
-	return []string{"docker", "cri-o", "containerd"}
+	return []string{"docker", "nvidia-docker", "cri-o", "containerd"}
 }
 
 // CommandRunner is the subset of command.Runner this package consumes
@@ -210,7 +210,7 @@ func New(c Config) (Manager, error) {
 	sm := sysinit.New(c.Runner)
 
 	switch c.Type {
-	case "", "docker":
+	case "", "docker", "nvidia-docker":
 		sp := c.Socket
 		cs := ""
 		// There is no more dockershim socket, in Kubernetes version 1.24 and beyond
@@ -219,6 +219,7 @@ func New(c Config) (Manager, error) {
 			cs = "cri-docker.socket"
 		}
 		return &Docker{
+			Type:              c.Type,
 			Socket:            sp,
 			Runner:            c.Runner,
 			NetworkPlugin:     c.NetworkPlugin,
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 044ce41c8..d083f94a3 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -67,6 +67,7 @@ func (e *ErrISOFeature) Error() string {
 
 // Docker contains Docker runtime state
 type Docker struct {
+	Type              string
 	Socket            string
 	Runner            CommandRunner
 	NetworkPlugin     string
@@ -156,7 +157,7 @@ func (r *Docker) Enable(disOthers bool, cgroupDriver string, inUserNamespace boo
 		klog.ErrorS(err, "Failed to enable", "service", "docker.socket")
 	}
 
-	if err := r.setCGroup(cgroupDriver); err != nil {
+	if err := r.configureDocker(cgroupDriver); err != nil {
 		return err
 	}
 
@@ -525,24 +526,51 @@ func (r *Docker) SystemLogCmd(len int) string {
 	return fmt.Sprintf("sudo journalctl -u docker -u cri-docker -n %d", len)
 }
 
-// setCGroup configures the docker daemon to use driver as cgroup manager
+type dockerDaemonConfig struct {
+	ExecOpts       []string              `json:"exec-opts"`
+	LogDriver      string                `json:"log-driver"`
+	LogOpts        dockerDaemonLogOpts   `json:"log-opts"`
+	StorageDriver  string                `json:"storage-driver"`
+	DefaultRuntime string                `json:"default-runtime,omitempty"`
+	Runtimes       *dockerDaemonRuntimes `json:"runtimes,omitempty"`
+}
+type dockerDaemonLogOpts struct {
+	MaxSize string `json:"max-size"`
+}
+type dockerDaemonRuntimes struct {
+	Nvidia struct {
+		Path        string        `json:"path"`
+		RuntimeArgs []interface{} `json:"runtimeArgs"`
+	} `json:"nvidia"`
+}
+
+// configureDocker configures the docker daemon to use driver as cgroup manager
 // ref: https://docs.docker.com/engine/reference/commandline/dockerd/#options-for-the-runtime
-func (r *Docker) setCGroup(driver string) error {
+func (r *Docker) configureDocker(driver string) error {
 	if driver == constants.UnknownCgroupDriver {
 		return fmt.Errorf("unable to configure docker to use unknown cgroup driver")
 	}
 
 	klog.Infof("configuring docker to use %q as cgroup driver...", driver)
-	daemonConfig := fmt.Sprintf(`{
-"exec-opts": ["native.cgroupdriver=%s"],
-"log-driver": "json-file",
-"log-opts": {
-	"max-size": "100m"
-},
-"storage-driver": "overlay2"
-}
-`, driver)
-	ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644")
+	daemonConfig := dockerDaemonConfig{
+		ExecOpts:  []string{"native.cgroupdriver=" + driver},
+		LogDriver: "json-file",
+		LogOpts: dockerDaemonLogOpts{
+			MaxSize: "100m",
+		},
+		StorageDriver: "overlay2",
+	}
+	if r.Type == "nvidia-docker" {
+		daemonConfig.DefaultRuntime = "nvidia"
+		runtimes := &dockerDaemonRuntimes{}
+		runtimes.Nvidia.Path = "/usr/bin/nvidia-container-runtime"
+		daemonConfig.Runtimes = runtimes
+	}
+	daemonConfigBytes, err := json.Marshal(daemonConfig)
+	if err != nil {
+		return err
+	}
+	ma := assets.NewMemoryAsset(daemonConfigBytes, "/etc/docker", "daemon.json", "0644")
 	return r.Runner.Copy(ma)
 }
 


Test patch:
diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go
index 60e068732..ed7e5d6b5 100644
--- a/pkg/minikube/cruntime/cruntime_test.go
+++ b/pkg/minikube/cruntime/cruntime_test.go
@@ -40,6 +40,7 @@ func TestName(t *testing.T) {
 	}{
 		{"", "Docker"},
 		{"docker", "Docker"},
+		{"nvidia-docker", "Docker"},
 		{"crio", "CRI-O"},
 		{"cri-o", "CRI-O"},
 		{"containerd", "containerd"},
@@ -124,6 +125,7 @@ func TestCGroupDriver(t *testing.T) {
 		want    string
 	}{
 		{"docker", "cgroupfs"},
+		{"nvidia-docker", "cgroupfs"},
 		{"crio", "cgroupfs"},
 		{"containerd", "cgroupfs"},
 	}
@@ -155,6 +157,12 @@ func TestKubeletOptions(t *testing.T) {
 		{"docker", "1.24.0", map[string]string{
 			"container-runtime-endpoint": "unix:///var/run/cri-dockerd.sock",
 		}},
+		{"nvidia-docker", "1.23.0", map[string]string{
+			"container-runtime": "docker",
+		}},
+		{"nvidia-docker", "1.25.0", map[string]string{
+			"container-runtime-endpoint": "unix:///var/run/cri-dockerd.sock",
+		}},
 		{"crio", "1.25.0", map[string]string{
 			"container-runtime-endpoint": "unix:///var/run/crio/crio.sock",
 		}},
@@ -680,6 +688,13 @@ func TestEnable(t *testing.T) {
 				"crio":          SvcExited,
 				"crio-shutdown": SvcExited,
 			}},
+		{"nvidia-docker", defaultServices,
+			map[string]serviceState{
+				"docker":        SvcRestarted,
+				"containerd":    SvcExited,
+				"crio":          SvcExited,
+				"crio-shutdown": SvcExited,
+			}},
 		{"containerd", defaultServices,
 			map[string]serviceState{
 				"docker":        SvcExited,
@@ -721,6 +736,7 @@ func TestContainerFunctions(t *testing.T) {
 		runtime string
 	}{
 		{"docker"},
+		{"nvidia-docker"},
 		{"crio"},
 		{"containerd"},
 	}
@@ -730,7 +746,7 @@ func TestContainerFunctions(t *testing.T) {
 		t.Run(tc.runtime, func(t *testing.T) {
 			runner := NewFakeRunner(t)
 			prefix := ""
-			if tc.runtime == "docker" {
+			if tc.runtime == "docker" || tc.runtime == "nvidia-docker" {
 				prefix = "k8s_"
 			}
 			runner.containers = map[string]string{