Change:
Kubernetes version (use ): v1.5.0-alpha.1.2189+-dirty docker: 1.12.3 Environment: fedora24 docker-engine-1.12.3-1.fc24.x86_64 What happened: if docker Cgroup Driver=cgroupfs, and set --cgroup-driver=systemd in kubelet, then enable --experimental-cgroups-per-qos=true; The container cgroup some is not correct. How to reproduce it (as minimally and precisely as possible): Anything else do we need to know: 1) docker Cgroup Driver=cgroupfs and --cgroup-driver=cgroupfs in kubelet works well 2) docker Cgroup Driver=systemd and --cgroup-driver=systemd in kubelet works well
help check, thanks.
In Kubernetes 1.6, we can work to detect this case and return an error. IMHO its a configuration error to have docker running with a cgroupfs driver paired with the kubelet running the systemd cgroup driver. /cc
We can also document the compatibility matrix for now.

Discussion:
"//pkg/apis/componentconfig:go_default_library", "//pkg/kubelet/api:go_default_library", "//pkg/kubelet/api/v1alpha1/runtime:go_default_library", <ins> "//pkg/kubelet/cm:go_default_library", </ins> "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/dockershim/cm:go_default_library", "//pkg/kubelet/dockertools:go_default_library",

// Apply cgroupsParent derived from the sandbox config. if lc := sandboxConfig.GetLinux(); lc != nil { // Apply Cgroup options. <del> // TODO: Check if this works with per-pod cgroups. // TODO: we need to pass the cgroup in syntax expected by cgroup driver but shim does not use docker info yet... hc.CgroupParent = lc.GetCgroupParent() </del> <ins> cgroupParent, err := ds.GenerateExpectedCgroupParent(lc.GetCgroupParent()) if err != nil { return "", fmt.Errorf("failed to generate cgroup parent in expected syntax for container %q: %v", config.Metadata.GetName(), err) } hc.CgroupParent = cgroupParent </ins> } // Set devices for container.

// applySandboxLinuxOptions applies LinuxPodSandboxConfig to dockercontainer.HostConfig and dockercontainer.ContainerCreateConfig. func (ds *dockerService) applySandboxLinuxOptions(hc *dockercontainer.HostConfig, lc *runtimeapi.LinuxPodSandboxConfig, createConfig *dockertypes.ContainerCreateConfig, image string) error { // Apply Cgroup options. <del> // TODO: Check if this works with per-pod cgroups. hc.CgroupParent = lc.GetCgroupParent() </del> <ins> cgroupParent, err := ds.GenerateExpectedCgroupParent(lc.GetCgroupParent()) if err != nil { return err } hc.CgroupParent = cgroupParent </ins> // Apply security context. applySandboxSecurityContext(lc, createConfig.Config, hc, ds.networkPlugin)

"k8s.io/kubernetes/pkg/apis/componentconfig" internalapi "k8s.io/kubernetes/pkg/kubelet/api" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" <ins> kubecm "k8s.io/kubernetes/pkg/kubelet/cm" </ins> kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/cm" "k8s.io/kubernetes/pkg/kubelet/dockertools"

var internalLabelKeys []string = []string{containerTypeLabelKey, containerLogPathLabelKey, sandboxIDLabelKey} // NOTE: Anything passed to DockerService should be eventually handled in another way when we switch to running the shim as a different process. <del> func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot string, podSandboxImage string, streamingConfig *streaming.Config, pluginSettings *NetworkPluginSettings, cgroupsName string) (DockerService, error) { </del> <ins> func NewDockerService(client dockertools.DockerInterface, seccompProfileRoot string, podSandboxImage string, streamingConfig *streaming.Config, pluginSettings *NetworkPluginSettings, cgroupsName string, kubeCgroupDriver string) (DockerService, error) { </ins> c := dockertools.NewInstrumentedDockerInterface(client) ds := &dockerService{ seccompProfileRoot: seccompProfileRoot,

} ds.networkPlugin = plug glog.Infof("Docker cri networking managed by %v", plug.Name()) <ins> // NOTE: cgroup driver is only detectable in docker 1.11+ var cgroupDriver string dockerInfo, err := ds.client.Info() if err != nil { glog.Errorf("failed to execute Info() call to the Docker client: %v", err) glog.Warningf("Using fallback default of cgroupfs as cgroup driver") } else { cgroupDriver = dockerInfo.CgroupDriver if len(kubeCgroupDriver) != 0 && kubeCgroupDriver != cgroupDriver { return nil, fmt.Errorf("misconfiguration: kubelet cgroup driver: %q is different from docker cgroup driver: %q", kubeCgroupDriver, cgroupDriver) } glog.Infof("Setting cgroupDriver to %s", cgroupDriver) } ds.cgroupDriver = cgroupDriver </ins> return ds, nil }

streamingServer    streaming.Server networkPlugin      network.NetworkPlugin containerManager   cm.ContainerManager <ins> // cgroup driver used by Docker runtime. cgroupDriver string </ins> } // Version returns the runtime name, runtime version and runtime API version

http.NotFound(w, r) } } <ins> // GenerateExpectedCgroupParent returns cgroup parent in syntax expected by cgroup driver func (ds *dockerService) GenerateExpectedCgroupParent(cgroupParent string) (string, error) { if len(cgroupParent) > 0 { // if docker uses the systemd cgroup driver, it expects *.slice style names for cgroup parent. // if we configured kubelet to use --cgroup-driver=cgroupfs, and docker is configured to use systemd driver // docker will fail to launch the container because the name we provide will not be a valid slice. // this is a very good thing. if ds.cgroupDriver == "systemd" { systemdCgroupParent, err := kubecm.ConvertCgroupFsNameToSystemd(cgroupParent) if err != nil { return "", err } cgroupParent = systemdCgroupParent } } glog.V(3).Infof("Setting cgroup parent to: %q", cgroupParent) return cgroupParent, nil } </ins>

// if there are any problems. dockerRoot := "/var/lib/docker" <del> // cgroup driver is only detectable in docker 1.12+ </del> <ins> // cgroup driver is only detectable in docker 1.11+ </ins> // when the execution driver is not detectable, we provide the cgroupfs form. // if your docker engine is configured to use the systemd cgroup driver, and you // want to use pod level cgroups, you must be on docker 1.12+ to ensure cgroup-parent

case "docker": streamingConfig := getStreamingConfig(kubeCfg, kubeDeps) // Use the new CRI shim for docker. <del> ds, err := dockershim.NewDockerService(klet.dockerClient, kubeCfg.SeccompProfileRoot, kubeCfg.PodInfraContainerImage, streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups) </del> <ins> ds, err := dockershim.NewDockerService(klet.dockerClient, kubeCfg.SeccompProfileRoot, kubeCfg.PodInfraContainerImage, streamingConfig, &pluginSettings, kubeCfg.RuntimeCgroups, kubeCfg.CgroupDriver) </ins> if err != nil { return nil, err }