Repository: kubernetes/minikube (go)

Issue:
minikube does not work with kubernetes 1.27 due to flag removal
The kubelet can't start, because the --container-runtime=remote flag has been dropped:

`"command failed" err="failed to parse kubelet flag: unknown flag: --container-runtime"`

It is only needed for versions before 1.24, and has been deprecated in 1.24 through 1.26

https://kubernetes.io/blog/2023/03/17/upcoming-changes-in-kubernetes-v1-27/

----

There is also a similar change, that one now needs to add `unix://` before the CRI sockets.

```
I0321 11:21:01.182927   11125 util_unix.go:103] "Using this endpoint is deprecated, please consider using full URL format" endpoint="/var/run/cri-dockerd.sock" URL="unix:///var/run/cri-dockerd.sock"
I0321 11:21:01.197552   11125 util_unix.go:103] "Using this endpoint is deprecated, please consider using full URL format" endpoint="/var/run/cri-dockerd.sock" URL="unix:///var/run/cri-dockerd.sock"
```

It is not needed to specify **both** endpoints, since the image endpoint will default to runtime:

```
runtime-endpoint: unix:///var/run/cri-dockerd.sock
image-endpoint: unix:///var/run/cri-dockerd.sock
```

But is mandatory to override them, since otherwise `kubeadm` will complain about the sockets.

`Found multiple CRI endpoints on the host. Please define which one do you wish to use by setting the 'criSocket' field in the kubeadm configuration file: unix:///var/run/containerd/containerd.sock, unix:///var/run/cri-dockerd.sock`

Previously it would look for dockershim.sock _before_  finding the containerd.sock (used by docker)

Pull request:
Remove container-runtime flag from later versions

The `--container-runtime` flag was deprecated in k8s 1.24, and removed in k8s 1.27

Add the extra `unix://` prefix everywhere, and remove the redundant endpoint config.

Closes #16112

Patch:
diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 4763e3759..58633d8f4 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -462,11 +462,13 @@ func (r *Containerd) CGroupDriver() (string, error) {
 
 // KubeletOptions returns kubelet options for a containerd
 func (r *Containerd) KubeletOptions() map[string]string {
-	return map[string]string{
-		"container-runtime":          "remote",
+	opts := map[string]string{
 		"container-runtime-endpoint": fmt.Sprintf("unix://%s", r.SocketPath()),
-		"image-service-endpoint":     fmt.Sprintf("unix://%s", r.SocketPath()),
 	}
+	if r.KubernetesVersion.LT(semver.MustParse("1.24.0-alpha.0")) {
+		opts["container-runtime"] = "remote"
+	}
+	return opts
 }
 
 // ListContainers returns a list of managed by this container runtime
diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go
index bfa4964f2..448099e5d 100644
--- a/pkg/minikube/cruntime/cri.go
+++ b/pkg/minikube/cruntime/cri.go
@@ -247,9 +247,7 @@ func stopCRIContainers(cr CommandRunner, ids []string) error {
 // populateCRIConfig sets up /etc/crictl.yaml
 func populateCRIConfig(cr CommandRunner, socket string) error {
 	cPath := "/etc/crictl.yaml"
-	tmpl := `runtime-endpoint: unix://{{.Socket}}
-image-endpoint: unix://{{.Socket}}
-`
+	tmpl := "runtime-endpoint: unix://{{.Socket}}\n"
 	t, err := template.New("crictl").Parse(tmpl)
 	if err != nil {
 		return err
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 0b38b200c..930aae5eb 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -358,11 +358,13 @@ func (r *CRIO) CGroupDriver() (string, error) {
 
 // KubeletOptions returns kubelet options for a runtime.
 func (r *CRIO) KubeletOptions() map[string]string {
-	return map[string]string{
-		"container-runtime":          "remote",
-		"container-runtime-endpoint": r.SocketPath(),
-		"image-service-endpoint":     r.SocketPath(),
+	opts := map[string]string{
+		"container-runtime-endpoint": fmt.Sprintf("unix://%s", r.SocketPath()),
 	}
+	if r.KubernetesVersion.LT(semver.MustParse("1.24.0-alpha.0")) {
+		opts["container-runtime"] = "remote"
+	}
+	return opts
 }
 
 // ListContainers returns a list of managed by this container runtime
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 2a200cd52..ca9453a09 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -382,11 +382,13 @@ func (r *Docker) CGroupDriver() (string, error) {
 // KubeletOptions returns kubelet options for a runtime.
 func (r *Docker) KubeletOptions() map[string]string {
 	if r.UseCRI {
-		return map[string]string{
-			"container-runtime":          "remote",
-			"container-runtime-endpoint": r.SocketPath(),
-			"image-service-endpoint":     r.SocketPath(),
+		opts := map[string]string{
+			"container-runtime-endpoint": fmt.Sprintf("unix://%s", r.SocketPath()),
 		}
+		if r.KubernetesVersion.LT(semver.MustParse("1.24.0-alpha.0")) {
+			opts["container-runtime"] = "remote"
+		}
+		return opts
 	}
 	return map[string]string{
 		"container-runtime": "docker",


Test patch:
diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go
index 56aa1cbe1..60e068732 100644
--- a/pkg/minikube/cruntime/cruntime_test.go
+++ b/pkg/minikube/cruntime/cruntime_test.go
@@ -148,30 +148,35 @@ func TestCGroupDriver(t *testing.T) {
 func TestKubeletOptions(t *testing.T) {
 	var tests = []struct {
 		runtime string
+		version string
 		want    map[string]string
 	}{
-		{"docker", map[string]string{"container-runtime": "docker"}},
-		{"crio", map[string]string{
-			"container-runtime":          "remote",
-			"container-runtime-endpoint": "/var/run/crio/crio.sock",
-			"image-service-endpoint":     "/var/run/crio/crio.sock",
+		{"docker", "1.23.0", map[string]string{"container-runtime": "docker"}},
+		{"docker", "1.24.0", map[string]string{
+			"container-runtime-endpoint": "unix:///var/run/cri-dockerd.sock",
 		}},
-		{"containerd", map[string]string{
+		{"crio", "1.25.0", map[string]string{
+			"container-runtime-endpoint": "unix:///var/run/crio/crio.sock",
+		}},
+		{"containerd", "1.23.0", map[string]string{
 			"container-runtime":          "remote",
 			"container-runtime-endpoint": "unix:///run/containerd/containerd.sock",
-			"image-service-endpoint":     "unix:///run/containerd/containerd.sock",
+		}},
+		{"containerd", "1.24.0", map[string]string{
+			"container-runtime-endpoint": "unix:///run/containerd/containerd.sock",
 		}},
 	}
 	for _, tc := range tests {
 		t.Run(tc.runtime, func(t *testing.T) {
-			r, err := New(Config{Type: tc.runtime})
+			kv := semver.MustParse(tc.version)
+			r, err := New(Config{Type: tc.runtime, KubernetesVersion: kv})
 			if err != nil {
-				t.Fatalf("New(%s): %v", tc.runtime, err)
+				t.Fatalf("New(%s, %s): %v", tc.runtime, tc.version, err)
 			}
 
 			got := r.KubeletOptions()
 			if diff := cmp.Diff(tc.want, got); diff != "" {
-				t.Errorf("KubeletOptions(%s) returned diff (-want +got):\n%s", tc.runtime, diff)
+				t.Errorf("KubeletOptions(%s, %s) returned diff (-want +got):\n%s", tc.runtime, tc.version, diff)
 			}
 		})
 	}